diff options
Diffstat (limited to 'doc')
736 files changed, 26768 insertions, 12171 deletions
diff --git a/doc/.vale/gitlab/BadgeCapitalization.yml b/doc/.vale/gitlab/BadgeCapitalization.yml new file mode 100644 index 00000000000..7e68a06b4d5 --- /dev/null +++ b/doc/.vale/gitlab/BadgeCapitalization.yml @@ -0,0 +1,42 @@ +--- +# Verifies that badges are not lower case, which won't render properly. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: existence +message: 'Badge "%s" must be capitalized.' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#product-badges +level: error +scope: raw +raw: + - '(\*\*\(Core\)\*\*|' + - '\*\*\(core\)\*\*|' + - '\*\*\(Starter\)\*\*|' + - '\*\*\(starter\)\*\*|' + - '\*\*\(Premium\)\*\*|' + - '\*\*\(premium\)\*\*|' + - '\*\*\(Ultimate\)\*\*|' + - '\*\*\(ultimate\)\*\*|' + - '\*\*\(Core Only\)\*\*|' + - '\*\*\(Core only\)\*\*|' + - '\*\*\(core only\)\*\*|' + - '\*\*\(Starter Only\)\*\*|' + - '\*\*\(Starter only\)\*\*|' + - '\*\*\(starter only\)\*\*|' + - '\*\*\(Premium Only\)\*\*|' + - '\*\*\(Premium only\)\*\*|' + - '\*\*\(premium only\)\*\*|' + - '\*\*\(Ultimate Only\)\*\*|' + - '\*\*\(Ultimate only\)\*\*|' + - '\*\*\(ultimate only\)\*\*|' + - '\*\*\(Free Only\)\*\*|' + - '\*\*\(Free only\)\*\*|' + - '\*\*\(free only\)\*\*|' + - '\*\*\(Bronze Only\)\*\*|' + - '\*\*\(Bronze only\)\*\*|' + - '\*\*\(bronze only\)\*\*|' + - '\*\*\(Silver Only\)\*\*|' + - '\*\*\(Silver only\)\*\*|' + - '\*\*\(silver only\)\*\*|' + - '\*\*\(Gold Only\)\*\*|' + - '\*\*\(Gold only\)\*\*|' + - '\*\*\(gold only\)\*\*)' diff --git a/doc/.vale/gitlab/British.yml b/doc/.vale/gitlab/British.yml new file mode 100644 index 00000000000..943e85beba1 --- /dev/null +++ b/doc/.vale/gitlab/British.yml @@ -0,0 +1,106 @@ +--- +# Checks for use of some of the top misused terms at GitLab. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: substitution +message: 'Use the American spelling "%s" instead of the British "%s".' +link: https://about.gitlab.com/handbook/communication/#top-misused-terms +level: error +ignorecase: true +swap: + aeon: eon + aeroplane: airplane + ageing: aging + aluminium: aluminum + anaemia: anemia + anaesthesia: anesthesia + analyse: analyze + annexe: annex + apologise: apologize + behaviour: behavior + busses: buses + calibre: caliber + centre: center + cheque: check + civilisation: civilization + civilise: civilize + colour: color + cosy: cozy + cypher: cipher + dependant: dependent + defence: defense + distil: distill + draught: draft + encyclopaedia: encyclopedia + enquiry: inquiry + enrol: enroll + enrolment: enrollment + enthral: enthrall + # equalled: equaled // Under discussion + # equalling: equaling // Under discussion + favourite: favorite + fibre: fiber + fillet: filet + flavour: flavor + furore: furor + fulfil: fulfill + gaol: jail + grey: gray + humour: humor + honour: honor + initialled: initialed + initialling: initialing + instil: instill + jewellery: jewelry + labelling: labeling + labelled: labeled + labour: labor + libellous: libelous + licence: license + likeable: likable + liveable: livable + lustre: luster + manoeuvre: maneuver + marvellous: marvelous + matt: matte + meagre: meager + metre: meter + mitre: miter + modelling: modeling + moustache: mustache + neighbour: neighbor + normalise: normalize + offence: offense + organise: organize + orientated: oriented + paralyse: paralyze + plough: plow + pretence: pretense + programme: program + pyjamas: pajamas + rateable: ratable + realise: realize + recognise: recognize + reconnoitre: reconnoiter + rumour: rumor + sabre: saber + saleable: salable + saltpetre: saltpeter + sceptic: skeptic + sepulchre: sepulcher + signalling: signaling + sizeable: sizable + skilful: skillful + sombre: somber + smoulder: smolder + speciality: specialty + spectre: specter + splendour: splendor + sulphur: sulfur + theatre: theater + travelled: traveled + traveller: traveler + travelling: traveling + unshakeable: unshakable + wilful: willful + yoghurt: yogurt diff --git a/doc/.vale/gitlab/Contractions.yml b/doc/.vale/gitlab/Contractions.yml index 5f389bd1ea4..f4ec24742da 100644 --- a/doc/.vale/gitlab/Contractions.yml +++ b/doc/.vale/gitlab/Contractions.yml @@ -3,7 +3,7 @@ # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: substitution -message: Use "%s" instead of "%s" in most cases. +message: Use "%s" instead of "%s", for a friendly, informal tone. link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language level: suggestion nonword: false diff --git a/doc/.vale/gitlab/Profanity.yml b/doc/.vale/gitlab/Profanity.yml new file mode 100644 index 00000000000..c386b23e52c --- /dev/null +++ b/doc/.vale/gitlab/Profanity.yml @@ -0,0 +1,30 @@ +--- +extends: existence +message: "Remove profanity: '%s'" +ignorecase: true +level: error +tokens: + - 'arse(hole)?' + - 'ass(hole)?' + - 'bastard' + - 'bitch' + - 'bloody' + - 'bollocks' + - 'bugger' + - 'cocksucker' + - 'crap' + - 'cunt' + - 'damn' + - 'eff(ing)?' + - 'fart' + - 'fuck(er|ing)?' + - 'goddamn(it?|ed)' + - 'hell' + - 'horseshit' + - 'motherfuck(ers?|ing)' + - 'piss(ing)?' + - 'shit' + - 'tits' + - 'turd' + - 'twat' + - 'wank(er|ing)?' diff --git a/doc/.vale/gitlab/ReferenceLinks.yml b/doc/.vale/gitlab/ReferenceLinks.yml new file mode 100644 index 00000000000..35a657710de --- /dev/null +++ b/doc/.vale/gitlab/ReferenceLinks.yml @@ -0,0 +1,10 @@ +# Checks for the presence of reference-style links that must be inline. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: existence +message: 'Link "%s" must be inline.' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#basic-link-criteria +level: error +scope: raw +raw: + - '\n\[.*\]: .*' diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml new file mode 100644 index 00000000000..fe690d708ed --- /dev/null +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -0,0 +1,16 @@ +--- +# Warns against using common shorthand for terms. +# For substitutions flagged as errors, see Substitutions.yml +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: substitution +message: 'If possible, use "%s" instead of "%s".' +link: https://about.gitlab.com/handbook/communication/#top-misused-terms +level: warning +ignorecase: true +swap: + admin: administrator + config: configuration + distro: distribution + info: information + repo: repository diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index 44b96d1a5e3..b9c0dbecf31 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -1,5 +1,6 @@ --- # Checks for use of some of the top misused terms at GitLab. +# For substitutions only flagged as warnings, see SubstitutionWarning.yml # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: substitution @@ -9,8 +10,16 @@ level: error ignorecase: true swap: GitLabber: GitLab team member + gitlab omnibus: Omnibus GitLab param: parameter params: parameters + pg: PostgreSQL postgres: PostgreSQL + raketask: Rake task + raketasks: Rake tasks + rspec: RSpec self hosted: self-managed self-hosted: self-managed + styleguide: style guide + x509: X.509 + yaml: YAML diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 3451219c267..c5e89f72043 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -55,6 +55,7 @@ CentOS Chatops Citrix Cloudwatch +Cobertura Cognito colocated colocating @@ -85,9 +86,11 @@ discoverability Disqus Dockerfile Dockerfiles +dotenv downvoted downvotes Dpl +Dreamweaver Elasticsearch enablement enqueued @@ -98,6 +101,7 @@ failovers failsafe favicon firewalled +Flawfinder Flowdock Fluentd Forgerock @@ -109,15 +113,20 @@ Gitea GitHub GitLab gitlabsos +Gitleaks Gitter Gmail Google +Gosec Gradle Grafana gravatar +Gzip hardcode hardcoded hardcodes +heatmap +heatmaps Helm Heroku Herokuish @@ -130,11 +139,13 @@ hotfixes hotfixing http https +idempotence Ingress initializer initializers interdependencies interdependency +interruptible Irker Istio jasmine-jquery @@ -152,6 +163,7 @@ Kibana Knative Kramdown Kubernetes +Kubesec Laravel LDAP Libravatar @@ -173,6 +185,7 @@ mergeable Microsoft middleware middlewares +Minikube MinIO mitmproxy misconfigure @@ -192,6 +205,8 @@ namespaced namespaces Nanoc NGINX +npm +Nurtch OAuth Okta offboarded @@ -204,10 +219,13 @@ Packagist parallelization parallelizations performant +Pipfile +Pipfiles Piwik PgBouncer plaintext PostgreSQL +precompile preconfigure preconfigured preconfigures @@ -232,10 +250,12 @@ Qualys Rackspace Raketask Raketasks +reachability rebase rebased rebases rebasing +Redcarpet Redis Redmine reCAPTCHA @@ -246,6 +266,7 @@ reindexed reindexes reindexing relicensing +remediations Repmgr Repmgrd requeue @@ -265,6 +286,7 @@ resync reverified reverifies reverify +Rubix runbook runbooks runit @@ -272,26 +294,36 @@ runtime runtimes Salesforce SAML +sbt Sendmail Sentry serverless Sidekiq sharding +shfmt Shibboleth sanitization serializer serializers serializing +Sitespeed Slack Slony SMTP -Sourcegraph +Sobelow +spidering Splunk +SpotBugs SSH storable strace +strikethrough +strikethroughs +subpath subfolder subfolders +subgraph +subgraphs sublicense sublicensed sublicenses @@ -305,6 +337,8 @@ subqueried subqueries subquery subquerying +substring +substrings syslog Tiller todos @@ -331,6 +365,7 @@ unchecking unchecks uncomment uncommented +uncommenting unencode unencoded unencoder @@ -349,20 +384,27 @@ unoptimize unoptimized unoptimizes unoptimizing +unprioritized unprotect unprotects unprotected unpublish unpublished unpublishes +unpublishing unreferenced unresolve unresolved unresolving +unschedule unstage unstaged unstages unstaging +unstash +unstashed +unstashing +untarred untracked untrusted unverified diff --git a/doc/README.md b/doc/README.md index 8437e7f798e..c9511b22f8f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -23,10 +23,11 @@ No matter how you use GitLab, we have documentation for you. | Essential Documentation | Essential Documentation | |:-------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------| | [**User Documentation**](user/index.md)<br/>Discover features and concepts for GitLab users. | [**Administrator documentation**](administration/index.md)<br/>Everything GitLab self-managed administrators need to know. | -| [**Contributing to GitLab**](#contributing-to-gitlab)<br/>At GitLab, everyone can contribute! | [**New to Git and GitLab?**](#new-to-git-and-gitlab)<br/>We have the resources to get you started. | +| [**Contributing to GitLab**](#contributing-to-gitlab)<br/>At GitLab, everyone can contribute! | [**New to Git and GitLab?**](#new-to-git-and-gitlab)<br/>We have the resources to get you started. | | [**Building an integration with GitLab?**](#building-an-integration-with-gitlab)<br/>Consult our automation and integration documentation. | [**Coming to GitLab from another platform?**](#coming-to-gitlab-from-another-platform)<br/>Consult our handy guides. | | [**Install GitLab**](https://about.gitlab.com/install/)<br/>Installation options for different platforms. | [**Customers**](subscriptions/index.md)<br/>Information for new and existing customers. | -| [**Update GitLab**](update/README.md)<br/>Update your GitLab self-managed instance to the latest version. | [**GitLab Releases**](https://about.gitlab.com/releases/)<br/>What's new in GitLab. | +| [**Update GitLab**](update/README.md)<br/>Update your GitLab self-managed instance to the latest version. | [**Reference Architectures**](administration/reference_architectures/index.md)<br/>GitLab's reference architectures | +| [**GitLab Releases**](https://about.gitlab.com/releases/)<br/>What's new in GitLab. | | ## Popular Documentation @@ -50,7 +51,7 @@ GitLab is the first single application for software development, security, and operations that enables [Concurrent DevOps](https://about.gitlab.com/concurrent-devops/), making the software lifecycle faster and radically improving the speed of business. -GitLab provides solutions for [all the stages of the DevOps lifecycle](https://about.gitlab.com/stages-devops-lifecycle/): +GitLab provides solutions for [each of the stages of the DevOps lifecycle](https://about.gitlab.com/stages-devops-lifecycle/):  @@ -65,7 +66,7 @@ The following sections provide links to documentation for each DevOps stage: |:------------------------|:------------------------------------------------------------| | [Manage](#manage) | Statistics and analytics features. | | [Plan](#plan) | Project planning and management features. | -| [Create](#create) | Source code and data creation and management features. | +| [Create](#create) | Source code, data creation, and management features. | | [Verify](#verify) | Testing, code quality, and continuous integration features. | | [Package](#package) | Docker container registry. | | [Release](#release) | Application release and delivery features. | @@ -118,6 +119,7 @@ The following documentation relates to the DevOps **Plan** stage: | [Project Issue Board](user/project/issue_board.md) | Display issues on a Scrum or Kanban board. | | [Quick Actions](user/project/quick_actions.md) | Shortcuts for common actions on issues or merge requests, replacing the need to click buttons or use dropdowns in GitLab's UI. | | [Related Issues](user/project/issues/related_issues.md) **(STARTER)** | Create a relationship between issues. | +| [Requirements Management](user/project/requirements/index.md) **(ULTIMATE)** | Check your products against a set of criteria. | | [Roadmap](user/group/roadmap/index.md) **(ULTIMATE)** | Visualize epic timelines. | | [Service Desk](user/project/service_desk.md) **(PREMIUM)** | A simple way to allow people to create issues in your GitLab instance without needing their own user account. | | [Time Tracking](user/project/time_tracking.md) | Track time spent on issues and merge requests. | @@ -283,8 +285,8 @@ The following documentation relates to the DevOps **Release** stage: | [Auto Deploy](topics/autodevops/stages.md#auto-deploy) | Configure GitLab for the deployment of your application. | | [Canary Deployments](user/project/canary_deployments.md) **(PREMIUM)** | Employ a popular CI strategy where a small portion of the fleet is updated to the new version first. | | [Deploy Boards](user/project/deploy_boards.md) **(PREMIUM)** | View the current health and status of each CI environment running on Kubernetes, displaying the status of the pods in the deployment. | -| [Environments and deployments](ci/environments.md) | With environments, you can control the continuous deployment of your software within GitLab. | -| [Environment-specific variables](ci/variables/README.md#limiting-environment-scopes-of-environment-variables) | Limit scope of variables to specific environments. | +| [Environments and deployments](ci/environments/index.md) | With environments, you can control the continuous deployment of your software within GitLab. | +| [Environment-specific variables](ci/variables/README.md#limit-the-environment-scopes-of-environment-variables) | Limit the scope of variables to specific environments. | | [GitLab CI/CD](ci/README.md) | Explore the features and capabilities of Continuous Deployment and Delivery with GitLab. | | [GitLab Pages](user/project/pages/index.md) | Build, test, and deploy a static site directly from GitLab. | | [Protected Runners](ci/runners/README.md#protected-runners) | Select Runners to only pick jobs for protected branches and tags. | @@ -299,7 +301,7 @@ The following documentation relates to the DevOps **Release** stage: ### Configure Automate your entire workflow from build to deploy and monitoring with GitLab -Auto DevOps. Best practice templates get you started with minimal to zero +Auto DevOps. Best practice templates help get you started with minimal to zero configuration. Then customize everything from buildpacks to CI/CD. The following documentation relates to the DevOps **Configure** stage: @@ -313,7 +315,7 @@ The following documentation relates to the DevOps **Configure** stage: | [Installing Applications](user/project/clusters/index.md#installing-applications) | Deploy Helm, Ingress, and Prometheus on Kubernetes. | | [Mattermost slash commands](user/project/integrations/mattermost_slash_commands.md) | Enable and use slash commands from within Mattermost. | | [Multiple Kubernetes Clusters](user/project/clusters/index.md#multiple-kubernetes-clusters-premium) **(PREMIUM)** | Associate more than one Kubernetes clusters to your project. | -| [Protected variables](ci/variables/README.md#protected-environment-variables) | Restrict variables to protected branches and tags. | +| [Protected variables](ci/variables/README.md#protect-a-custom-variable) | Restrict variables to protected branches and tags. | | [Serverless](user/project/clusters/serverless/index.md) | Run serverless workloads on Kubernetes. | | [Slack slash commands](user/project/integrations/slack_slash_commands.md) | Enable and use slash commands from within Slack. | | [Manage your infrastructure with Terraform](user/infrastructure/index.md) | Manage your infrastructure as you run your CI/CD pipeline. | @@ -335,7 +337,7 @@ The following documentation relates to the DevOps **Monitor** stage: | Monitor Topics | Description | |:------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------| -| [GitLab Performance Monitoring](administration/monitoring/performance/index.md) **(CORE ONLY)** | Use InfluxDB and Grafana to monitor the performance of your GitLab instance (will be eventually replaced by Prometheus). | +| [GitLab Performance Monitoring](administration/monitoring/performance/index.md) **(CORE ONLY)** | Use Prometheus and Grafana to monitor the performance of your GitLab instance. | | [GitLab Prometheus](administration/monitoring/prometheus/index.md) **(CORE ONLY)** | Configure the bundled Prometheus to collect various metrics from your GitLab instance. | | [Health check](user/admin_area/monitoring/health_check.md) | GitLab provides liveness and readiness probes to indicate service health and reachability to required services. | | [Prometheus project integration](user/project/integrations/prometheus.md) | Configure the Prometheus integration per project and monitor your CI/CD environments. | diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 26b4434de77..e42982e6524 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -1,6 +1,12 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Audit Events **(STARTER)** -GitLab offers a way to view the changes made within the GitLab server for owners and administrators on a [paid plan][ee]. +GitLab offers a way to view the changes made within the GitLab server for owners and administrators on a [paid plan](https://about.gitlab.com/pricing/). GitLab system administrators can also take advantage of the logs located on the filesystem. See [the logs system documentation](logs.md) for more details. @@ -28,10 +34,16 @@ There are two kinds of events logged: - Instance events scoped to the whole GitLab instance, used by your Compliance team to perform formal audits. +### Impersonation data **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/536) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. + +Impersonation is where an administrator uses credentials to perform an action as a different user. + ### Group events **(STARTER)** NOTE: **Note:** -You need Owner [permissions] to view the group Audit Events page. +You need Owner [permissions](../user/permissions.md) to view the group Audit Events page. To view a group's audit events, navigate to **Group > Settings > Audit Events**. From there, you can see the following actions: @@ -40,14 +52,15 @@ From there, you can see the following actions: - Group repository size limit changed - Group created or deleted - Group changed visibility -- User was added to group and with which [permissions] +- User was added to group and with which [permissions](../user/permissions.md) - User sign-in via [Group SAML](../user/group/saml_sso/index.md) - Permissions changes of a user assigned to a group - Removed user from group +- Project imported in to group - Project added to group and with which visibility level - Project removed from group - [Project shared with group](../user/project/members/share_project_with_groups.md) - and with which [permissions] + and with which [permissions](../user/permissions.md) - Removal of a previously shared group with a project - LFS enabled or disabled - Shared runners minutes limit changed @@ -61,7 +74,7 @@ Group events can also be accessed via the [Group Audit Events API](../api/audit_ ### Project events **(STARTER)** NOTE: **Note:** -You need Maintainer [permissions] or higher to view the project Audit Events page. +You need Maintainer [permissions](../user/permissions.md) or higher to view the project Audit Events page. To view a project's audit events, navigate to **Project > Settings > Audit Events**. From there, you can see the following actions: @@ -69,7 +82,7 @@ From there, you can see the following actions: - Added or removed deploy keys - Project created, deleted, renamed, moved(transferred), changed path - Project changed visibility level -- User was added to project and with which [permissions] +- User was added to project and with which [permissions](../user/permissions.md) - Permission changes of a user assigned to a project - User was removed from project - Project export was downloaded @@ -86,7 +99,7 @@ From there, you can see the following actions: ### Instance events **(PREMIUM ONLY)** -> [Introduced][ee-2336] in [GitLab Premium][ee] 9.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2336) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.3. Server-wide audit logging introduces the ability to observe user actions across the entire instance of your GitLab server, making it easy to understand who @@ -157,7 +170,3 @@ the steps bellow. ```ruby Feature.enable(:repository_push_audit_event) ``` - -[ee-2336]: https://gitlab.com/gitlab-org/gitlab/issues/2336 -[ee]: https://about.gitlab.com/pricing/ -[permissions]: ../user/permissions.md diff --git a/doc/administration/auditor_users.md b/doc/administration/auditor_users.md index 2bb229d4f68..ace210183b2 100644 --- a/doc/administration/auditor_users.md +++ b/doc/administration/auditor_users.md @@ -1,6 +1,6 @@ # Auditor users **(PREMIUM ONLY)** ->[Introduced][ee-998] in [GitLab Premium][eep] 8.17. +>[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/998) in [GitLab Premium](https://about.gitlab.com/pricing/) 8.17. Auditor users are given read-only access to all projects, groups, and other resources on the GitLab instance. @@ -12,7 +12,7 @@ snippets, etc.), and read-only access to **all** other resources, except the Admin Area. To put another way, they are just regular users (who can be added to projects, create personal snippets, create milestones on their groups, etc.) who also happen to have read-only access to all projects on the system that -they haven't been explicitly [given access][permissions] to. +they haven't been explicitly [given access](../user/permissions.md) to. The Auditor role is _not_ a read-only version of the Admin role. Auditor users will not be able to access the project/group settings pages, or the Admin Area. @@ -25,7 +25,7 @@ To sum up, assuming you have logged-in as an Auditor user: - For a project the Auditor owns, the Auditor should have full access to everything. - For a project the Auditor has been added to as a member, the Auditor should - have the same access as the [permissions] they were given to. For example, if + have the same access as the [permissions](../user/permissions.md) they were given to. For example, if they were added as a Developer, they could then push commits or comment on issues. - The Auditor cannot view the Admin Area, or perform any admin actions. @@ -82,7 +82,3 @@ instance, with the following permissions/restrictions: - Cannot create/modify files from the Web UI - Cannot merge a merge request - Cannot create project snippets - -[ee-998]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/998 -[eep]: https://about.gitlab.com/pricing/ -[permissions]: ../user/permissions.md diff --git a/doc/administration/auth/crowd.md b/doc/administration/auth/crowd.md index 6c2e4edac31..71938d4fd2b 100644 --- a/doc/administration/auth/crowd.md +++ b/doc/administration/auth/crowd.md @@ -66,14 +66,11 @@ Authenticate to GitLab using the Atlassian Crowd OmniAuth provider. 1. Change `YOUR_APP_NAME` to the application name from Crowd applications page. 1. Change `YOUR_APP_PASSWORD` to the application password you've set. 1. Save the configuration file. -1. [Reconfigure][] or [restart][] for the changes to take effect if you +1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart](../restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Crowd tab in the sign in form. -[reconfigure]: ../restart_gitlab.md#omnibus-gitlab-reconfigure -[restart]: ../restart_gitlab.md#installations-from-source - ## Troubleshooting If you see an error message like the one below when you sign in after Crowd authentication is configured, you may want to consult the Crowd administrator for the Crowd log file to know the exact cause: diff --git a/doc/administration/auth/google_secure_ldap.md b/doc/administration/auth/google_secure_ldap.md index cb7901ea5b4..b643dd2f7b9 100644 --- a/doc/administration/auth/google_secure_ldap.md +++ b/doc/administration/auth/google_secure_ldap.md @@ -133,7 +133,7 @@ values obtained during the LDAP client configuration earlier: EOS ``` -1. Save the file and [reconfigure] GitLab for the changes to take effect. +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. --- @@ -204,10 +204,7 @@ values obtained during the LDAP client configuration earlier: -----END PRIVATE KEY----- ``` -1. Save the file and [restart] GitLab for the changes to take effect. - -[reconfigure]: ../restart_gitlab.md#omnibus-gitlab-reconfigure -[restart]: ../restart_gitlab.md#installations-from-source +1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. <!-- ## Troubleshooting diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md b/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md index 49bf786061e..01da4528eab 100644 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md +++ b/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md @@ -45,10 +45,7 @@ Configuring organizational units (**OU**s) is an important part of setting up Ac | GitLab **OU** Design | GitLab AD Structure | | :----------------------------: | :------------------------------: | -| ![GitLab OU Design][gitlab_ou] | ![GitLab AD Structure][ldap_ou] | - -[gitlab_ou]: img/gitlab_ou.png -[ldap_ou]: img/ldap_ou.gif +|  |  | Using PowerShell you can output the **OU** structure as a table (_all names are examples only_): diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md index 46bc079971d..d7b32c8a92a 100644 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md +++ b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md @@ -8,7 +8,7 @@ This article expands on [How to Configure LDAP with GitLab CE](../how_to_configu ## GitLab Enterprise Edition - LDAP features -[GitLab Enterprise Edition (EE)](https://about.gitlab.com/pricing/) has a number of advantages when it comes to integrating with Active Directory (LDAP): +[GitLab Enterprise Edition (EE)](https://about.gitlab.com/pricing/) has several advantages when it comes to integrating with Active Directory (LDAP): - [Administrator Sync](../ldap-ee.md#administrator-sync): As an extension of group sync, you can automatically manage your global GitLab administrators. Specify a group CN for `admin_group` and all members of the LDAP group will be given administrator privileges. - [Group Sync](#group-sync): This allows GitLab group membership to be automatically updated based on LDAP group members. @@ -16,7 +16,7 @@ This article expands on [How to Configure LDAP with GitLab CE](../how_to_configu - Daily user synchronization: Once a day, GitLab will run a synchronization to check and update GitLab users against LDAP. This process updates all user details automatically. -On the following section, you'll find a description for each of these features. Read through [LDAP GitLab EE docs](../ldap-ee.md) for complementary information. +In the following section, you'll find a description of each of these features. Read through [LDAP GitLab EE docs](../ldap-ee.md) for complementary information.  @@ -28,7 +28,7 @@ Group syncing allows AD (LDAP) groups to be mapped to GitLab groups. This provid #### Creating group links - example -As an example, let's suppose we have a "UKGov" GitLab group, which deals with confidential government information. Therefore, it is important that users of this group are given the correct permissions to projects contained within the group. Granular group permissions can be applied based on the AD group. +As an example, let's suppose we have a "UKGov" GitLab group, which deals with confidential government information. Therefore, users of this group must be given the correct permissions to projects contained within the group. Granular group permissions can be applied based on the AD group. **UK Developers** of our "UKGov" group are given **"developer"** permissions. diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md index 15eee50c771..5a773485842 100644 --- a/doc/administration/auth/jwt.md +++ b/doc/administration/auth/jwt.md @@ -64,7 +64,7 @@ JWT will provide you with a secret key for you to use. 1. Change `YOUR_APP_SECRET` to the client secret and set `auth_url` to your redirect URL. 1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you +1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a JWT icon below the regular sign in form. @@ -72,9 +72,6 @@ Click the icon to begin the authentication process. JWT will ask the user to sign in and authorize the GitLab application. If everything goes well, the user will be redirected to GitLab and will be signed in. -[reconfigure]: ../restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../restart_gitlab.md#installations-from-source - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/administration/auth/ldap-ee.md b/doc/administration/auth/ldap-ee.md index 3bc30c4c01c..9e193dba08c 100644 --- a/doc/administration/auth/ldap-ee.md +++ b/doc/administration/auth/ldap-ee.md @@ -119,6 +119,9 @@ following. To take advantage of group sync, group owners or maintainers will need to [create one or more LDAP group links](#adding-group-links). +NOTE: **Note:** +If an LDAP user is a group member when LDAP Synchronization is added, and they are not part of the LDAP group, they will be removed from the group. + ### Adding group links Once [group sync has been configured](#group-sync) on the instance, one or more LDAP diff --git a/doc/administration/auth/ldap-troubleshooting.md b/doc/administration/auth/ldap-troubleshooting.md index 612f782e841..77c9c30d140 100644 --- a/doc/administration/auth/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap-troubleshooting.md @@ -312,7 +312,7 @@ things to check to debug the situation. interval](ldap-ee.md#adjusting-ldap-group-sync-schedule) for the group to sync. To speed up the process, either go to the GitLab group **Settings -> Members** and press **Sync now** (sync one group) or [run the group sync Rake - task](../raketasks/ldap.md#run-a-group-sync) (sync all groups). + task](../raketasks/ldap.md#run-a-group-sync-starter-only) (sync all groups). If all of the above looks good, jump in to a little more advanced debugging in the rails console. @@ -352,7 +352,7 @@ GitLab syncs the `admin_group`. NOTE: **NOTE:** To sync all groups manually when debugging is unnecessary, [use the Rake -task](../raketasks/ldap.md#run-a-group-sync) instead. +task](../raketasks/ldap.md#run-a-group-sync-starter-only) instead. The output from a manual [group sync](ldap-ee.md#group-sync) can show you what happens when GitLab syncs its LDAP group memberships against LDAP. diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index 76e83c8e0ad..7a808636e94 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -94,7 +94,7 @@ The OpenID Connect will provide you with a client details and secret for you to - `basic` - HTTP Basic Authentication - `jwt_bearer` - JWT based authentication (private key and client secret signing) - `mtls` - Mutual TLS or X.509 certificate validation - - Any other value will POST the client id and secret in the request body + - Any other value will POST the client ID and secret in the request body - If not specified, defaults to `basic`. - `<uid_field>` (optional) is the field name from the `user_info` details that will be used as `uid` value. For example, `preferred_username`. If this value is not provided or the field with the configured value is missing from the `user_info` details, the `uid` will use the `sub` field. diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index 8986a9866e7..7131fd7571f 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -37,7 +37,7 @@ To use a smartcard with an X.509 certificate to authenticate against a local database with GitLab, `CN` and `emailAddress` must be defined in the certificate. For example: -```text +```plaintext Certificate: Data: Version: 1 (0x0) @@ -73,7 +73,7 @@ database with GitLab, in: For example: -```text +```plaintext Certificate: Data: Version: 1 (0x0) diff --git a/doc/administration/availability/index.md b/doc/administration/availability/index.md index a0d4ea7919f..748373c8941 100644 --- a/doc/administration/availability/index.md +++ b/doc/administration/availability/index.md @@ -1,139 +1,5 @@ --- -type: reference, concepts +redirect_to: ../reference_architectures/index.md --- -# Availability - -GitLab offers high availability options for organizations that require -the fault tolerance and redundancy necessary to maintain high-uptime operations. - -Please consult our [scaling documentation](../scaling) if you want to resolve -performance bottlenecks you encounter in individual GitLab components without -incurring the additional complexity costs associated with maintaining a -highly-available architecture. - -On this page, we present examples of self-managed instances which demonstrate -how GitLab can be scaled out and made highly available. These examples progress -from simple to complex as scaling or highly-available components are added. - -For larger setups serving 2,000 or more users, we provide -[reference architectures](../scaling/index.md#reference-architectures) based on GitLab's -experience with GitLab.com and internal scale testing that aim to achieve the -right balance of scalability and availability. - -For detailed insight into how GitLab scales and configures GitLab.com, you can -watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac) -with [John Northrup](https://gitlab.com/northrup), and live questions coming -in from some of our customers. - -GitLab offers a number of options to manage availability and resiliency. Below are the options to consider with trade-offs. - -| Event | GitLab Feature | Recovery Point Objective (RPO) | Recovery Time Objective (RTO) | Cost | -| ----- | -------------- | --- | --- | ---- | -| Availability Zone failure | "GitLab HA" | No loss | No loss | 2x Git storage, multiple nodes balanced across AZ's | -| Region failure | "GitLab Disaster Recovery" | 5-10 minutes | 30 minutes | 2x primary cost | -| All failures | Backup/Restore | Last backup | Hours to Days | Cost of storing the backups | - -## High availability - -### Omnibus installation with automatic database failover - -By adding automatic failover for database systems, we can enable higher uptime with an additional layer of complexity. - -- For PostgreSQL, we provide repmgr for server cluster management and failover - and a combination of [PgBouncer](../high_availability/pgbouncer.md) and [Consul](../high_availability/consul.md) for - database client cutover. -- For Redis, we use [Redis Sentinel](../high_availability/redis.md) for server failover and client cutover. - -You can also optionally run [additional Sidekiq processes on dedicated hardware](../high_availability/sidekiq.md) -and configure individual Sidekiq processes to -[process specific background job queues](../operations/extra_sidekiq_processes.md) -if you need to scale out background job processing. - -### GitLab Geo - -GitLab Geo allows you to replicate your GitLab instance to other geographical locations as a read-only fully operational instance that can also be promoted in case of disaster. - -This configuration is supported in [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/). - -References: - -- [Geo Documentation](../geo/replication/index.md) -- [GitLab Geo with a highly available configuration](../geo/replication/high_availability.md) - -## GitLab components and configuration instructions - -The GitLab application depends on the following [components](../../development/architecture.md#component-diagram). -It can also depend on several third party services depending on -your environment setup. Here we'll detail both in the order in which -you would typically configure them along with our recommendations for -their use and configuration. - -### Third party services - -Here's some details of several third party services a typical environment -will depend on. The services can be provided by numerous applications -or providers and further advice can be given on how best to select. -These should be configured first, before the [GitLab components](#gitlab-components). - -| Component | Description | Configuration instructions | -|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| -| [Load Balancer(s)](../high_availability/load_balancer.md)[^6] | Handles load balancing for the GitLab nodes where required | [Load balancer HA configuration](../high_availability/load_balancer.md) | -| [Cloud Object Storage service](../high_availability/object_storage.md)[^4] | Recommended store for shared data objects | [Cloud Object Storage configuration](../high_availability/object_storage.md) | -| [NFS](../high_availability/nfs.md)[^5] [^7] | Shared disk storage service. Can be used as an alternative for Gitaly or Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | - -### GitLab components - -Next are all of the components provided directly by GitLab. As mentioned -earlier, they are presented in the typical order you would configure -them. - -| Component | Description | Configuration instructions | -|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|---------------------------------------------------------------| -| [Consul](../../development/architecture.md#consul)[^3] | Service discovery and health checks/failover | [Consul HA configuration](../high_availability/consul.md) **(PREMIUM ONLY)** | -| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [Database HA configuration](../high_availability/database.md) | -| [PgBouncer](../../development/architecture.md#pgbouncer) | Database Pool Manager | [PgBouncer HA configuration](../high_availability/pgbouncer.md) **(PREMIUM ONLY)** | -| [Redis](../../development/architecture.md#redis)[^3] with Redis Sentinel | Key/Value store for shared data with HA watcher service | [Redis HA configuration](../high_availability/redis.md) | -| [Gitaly](../../development/architecture.md#gitaly)[^2] [^5] [^7] | Recommended high-level storage for Git repository data | [Gitaly HA configuration](../high_availability/gitaly.md) | -| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/Background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | -| [GitLab application nodes](../../development/architecture.md#unicorn)[^1] | (Unicorn / Puma, Workhorse) - Web-requests (UI, API, Git over HTTP) | [GitLab app HA/scaling configuration](../high_availability/gitlab.md) | -| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling/HA](../high_availability/monitoring_node.md) | - -In some cases, components can be combined on the same nodes to reduce complexity as well. - -[^1]: In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with 4 threads. - -[^2]: Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend 2 nodes as an absolute minimum for HA environments - and at least 4 nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -[^3]: Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (up to 5,000 users) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately as well for each Redis Cluster. - -[^4]: For data objects such as LFS, Uploads, Artifacts, etc. We recommend a [Cloud Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -[^5]: NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). - -[^6]: Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. However other reputable load balancers with similar feature sets - should also work instead but be aware these aren't validated. - -[^7]: We strongly recommend that any Gitaly and / or NFS nodes are set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. +This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/database_load_balancing.md b/doc/administration/database_load_balancing.md index b6ce6883dd6..0f566fcc114 100644 --- a/doc/administration/database_load_balancing.md +++ b/doc/administration/database_load_balancing.md @@ -1,6 +1,6 @@ # Database Load Balancing **(PREMIUM ONLY)** -> [Introduced][ee-1283] in [GitLab Premium][eep] 9.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1283) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. Distribute read-only queries among multiple database servers. @@ -11,7 +11,7 @@ multiple computing resources. Load balancing aims to optimize resource use, maximize throughput, minimize response time, and avoid overload of any single resource. Using multiple components with load balancing instead of a single component may increase reliability and availability through redundancy. -[_Wikipedia article_][wikipedia] +[_Wikipedia article_](https://en.wikipedia.org/wiki/Load_balancing_(computing)) When database load balancing is enabled in GitLab, the load is balanced using a simple round-robin algorithm, without any external dependencies such as Redis. @@ -26,9 +26,9 @@ sent to the primary (unless necessary), the primary (`db3`) hardly has any load. ## Requirements -For load balancing to work you will need at least PostgreSQL 9.2 or newer, -[**MySQL is not supported**][db-req]. You also need to make sure that you have -at least 1 secondary in [hot standby][hot-standby] mode. +For load balancing to work you will need at least PostgreSQL 11 or newer, +[**MySQL is not supported**](../install/requirements.md#database). You also need to make sure that you have +at least 1 secondary in [hot standby](https://www.postgresql.org/docs/11/hot-standby.html) mode. Load balancing also requires that the configured hosts **always** point to the primary, even after a database failover. Furthermore, the additional hosts to @@ -78,7 +78,7 @@ the following. This will balance the load between `host1.example.com` and gitlab_rails['db_load_balancing'] = { 'hosts' => ['host1.example.com', 'host2.example.com'] } ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. --- @@ -97,11 +97,11 @@ the following. This will balance the load between `host1.example.com` and - host2.example.com ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ## Service Discovery -> [Introduced][ee-5883] in [GitLab Premium][eep] 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5883) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. Service discovery allows GitLab to automatically retrieve a list of secondary databases to use, instead of having to manually specify these in the @@ -161,12 +161,16 @@ When the list of hosts is updated, it might take a while for the old connections to be terminated. The `disconnect_timeout` setting can be used to enforce an upper limit on the time it will take to terminate all old database connections. -Some nameservers (like [Consul][consul-udp]) can return a truncated list of hosts when +Some nameservers (like [Consul](https://www.consul.io/docs/agent/dns.html#udp-based-dns-queries)) can return a truncated list of hosts when queried over UDP. To overcome this issue, you can use TCP for querying by setting `use_tcp` to `true`. ### Forking +NOTE: **Note:** +Starting with GitLab 13.0, Puma is the default web server used in GitLab +all-in-one package based installations as well as GitLab Helm chart deployments. + If you use an application server that forks, such as Unicorn, you _have to_ update your Unicorn configuration to start service discovery _after_ a fork. Failure to do so will lead to service discovery only running in the parent @@ -239,14 +243,14 @@ For example: ## Handling Stale Reads -> [Introduced][ee-3526] in [GitLab Premium][eep] 10.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3526) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.3. To prevent reading from an outdated secondary the load balancer will check if it is in sync with the primary. If the data is determined to be recent enough the secondary can be used, otherwise it will be ignored. To reduce the overhead of these checks we only perform these checks at certain intervals. -There are three configuration options that influence this behaviour: +There are three configuration options that influence this behavior: | Option | Description | Default | |------------------------------|----------------------------------------------------------------------------------------------------------------|------------| @@ -270,14 +274,3 @@ production: max_replication_lag_time: 30 replica_check_interval: 30 ``` - -[hot-standby]: https://www.postgresql.org/docs/9.6/hot-standby.html -[ee-1283]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1283 -[eep]: https://about.gitlab.com/pricing/ -[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" -[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab" -[wikipedia]: https://en.wikipedia.org/wiki/Load_balancing_(computing) -[db-req]: ../install/requirements.md#database -[ee-3526]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3526 -[ee-5883]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5883 -[consul-udp]: https://www.consul.io/docs/agent/dns.html#udp-based-dns-queries diff --git a/doc/administration/external_database.md b/doc/administration/external_database.md index ec2d30c82d1..47509828c20 100644 --- a/doc/administration/external_database.md +++ b/doc/administration/external_database.md @@ -5,7 +5,7 @@ managed service for PostgreSQL. For example, AWS offers a managed Relational Database Service (RDS) that runs PostgreSQL. Alternatively, you may opt to manage your own PostgreSQL instance or cluster -separate from the GitLab Omnibus package. +separate from the Omnibus GitLab package. If you use a cloud-managed service, or provide your own PostgreSQL instance: @@ -13,5 +13,29 @@ If you use a cloud-managed service, or provide your own PostgreSQL instance: [database requirements document](../install/requirements.md#database). 1. Set up a `gitlab` username with a password of your choice. The `gitlab` user needs privileges to create the `gitlabhq_production` database. -1. Configure the GitLab application servers with the appropriate details. - This step is covered in [Configuring GitLab for HA](high_availability/gitlab.md). +1. If you are using a cloud-managed service, you may need to grant additional + roles to your `gitlab` user: + - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. + - Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. + +1. Configure the GitLab application servers with the appropriate connection details + for your external PostgreSQL service in your `/etc/gitlab/gitlab.rb` file: + + ```ruby + # Disable the bundled Omnibus provided PostgreSQL + postgresql['enable'] = false + + # PostgreSQL connection details + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server + gitlab_rails['db_password'] = 'DB password' + ``` + + For more information on GitLab HA setups, refer to [configuring GitLab for HA](high_availability/gitlab.md). + +1. Reconfigure for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md index 19d4de3b705..489d1924803 100644 --- a/doc/administration/external_pipeline_validation.md +++ b/doc/administration/external_pipeline_validation.md @@ -20,7 +20,7 @@ Response Code Legend: ## Configuration -Set the `EXTERNAL_VALIDATION_SERVICE_URL` to the external service url. +Set the `EXTERNAL_VALIDATION_SERVICE_URL` to the external service URL. ## Payload Schema diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md new file mode 100644 index 00000000000..59cd5497032 --- /dev/null +++ b/doc/administration/feature_flags.md @@ -0,0 +1,99 @@ +--- +type: reference +description: "GitLab administrator: enable and disable GitLab features deployed behind feature flags" +--- + +# Enable and disable GitLab features deployed behind feature flags **(CORE ONLY)** + +GitLab adopted [feature flags strategies](../development/feature_flags/index.md) +to deploy features in an early stage of development so that they can be +incrementally rolled out. + +Before making them permanently available, features can be deployed behind +flags for a [number of reasons](../development/feature_flags/process.md#when-to-use-feature-flags), such as: + +- To test the feature. +- To get feedback from users and customers while in an early stage of the development of the feature. +- To evaluate users adoption. +- To evaluate how it impacts GitLab's performance. +- To build it in smaller pieces throughout releases. + +Features behind flags can be gradually rolled out, typically: + +1. The feature starts disabled by default. +1. The feature becomes enabled by default. +1. The feature flag is removed. + +These features can be enabled and disabled to allow or disallow users to use +them. It can be done by GitLab administrators with access to GitLab Rails +console. + +If you used a certain feature and identified a bug, a misbehavior, or an +error, it's very important that you [**provide feedback**](https://gitlab.com/gitlab-org/gitlab/issues/new?issue[title]=Docs%20-%20feature%20flag%20feedback%3A%20Feature%20Name&issue[description]=Describe%20the%20problem%20you%27ve%20encountered.%0A%0A%3C!--%20Don%27t%20edit%20below%20this%20line%20--%3E%0A%0A%2Flabel%20~%22docs%5C-comments%22%20) to GitLab as soon +as possible so we can improve or fix it while behind a flag. When you upgrade +GitLab to an earlier version, the feature flag status may change. + +NOTE: **Note:** +Mind that features deployed behind feature flags may not be ready for +production use. However, disabling features behind flags that were deployed +enabled by default may also present a risk. If they're enabled, we recommend +you leave them as-is. + +## How to enable and disable features behind flags + +Each feature has its own flag that should be used to enable and disable it. +The documentation of each feature behind a flag includes a section informing +the status of the flag and the command to enable or disable it. + +### Start the GitLab Rails console + +The first thing you need to enable or disable a feature behind a flag is to +start a session on GitLab Rails console. + +For Omnibus installations: + +```shell +sudo gitlab-rails console +``` + +For installations from the source: + +```shell +sudo -u git -H bundle exec rails console -e production +``` + +For details, see [starting a Rails console session](troubleshooting/debug.md#starting-a-rails-console-session). + +### Enable or disable the feature + +Once the Rails console session has started, run the `Feature.enable` or +`Feature.disable` commands accordingly. The specific flag can be found +in the feature's documentation itself. + +To enable a feature, run: + +```ruby +Feature.enable(:<feature flag>) +``` + +Example, to enable Evidence Collection: + +```ruby +Feature.enable(:release_evidence_collection) +``` + +To disable a feature, run: + +```ruby +Feature.disable(:<feature flag>) +``` + +Example, to disable Evidence Collection: + +```ruby +Feature.disable(:release_evidence_collection) +``` + +When the feature is ready, GitLab will remove the feature flag, the option for +enabling and disabling it will no longer exist, and the feature will become +available in all instances. diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index 21ade36a2a5..7903da675fd 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -14,12 +14,12 @@ ensure functionality is preserved across versions and covered by tests. NOTE: **Note:** File hooks must be configured on the filesystem of the GitLab server. Only GitLab server administrators will be able to complete these tasks. Explore -[system hooks] or [webhooks] as an option if you do not have filesystem access. +[system hooks](../system_hooks/system_hooks.md) or [webhooks](../user/project/integrations/webhooks.md) as an option if you do not have filesystem access. A file hook will run on each event so it's up to you to filter events or projects within a file hook code. You can have as many file hooks as you want. Each file hook will be triggered by GitLab asynchronously in case of an event. For a list of events -see the [system hooks] documentation. +see the [system hooks](../system_hooks/system_hooks.md) documentation. ## Setup @@ -35,8 +35,8 @@ Follow the steps below to set up a custom hook: `/home/git/gitlab/file_hooks/`. For Omnibus installs the path is usually `/opt/gitlab/embedded/service/gitlab-rails/file_hooks`. - For [highly available] configurations, your hook file should exist on each - application server. + For [configurations with multiple servers](reference_architectures/index.md), + your hook file should exist on each application server. 1. Inside the `file_hooks` directory, create a file with a name of your choice, without spaces or special characters. @@ -46,7 +46,7 @@ Follow the steps below to set up a custom hook: language type. For example, if the script is in Ruby the shebang will probably be `#!/usr/bin/env ruby`. 1. The data to the file hook will be provided as JSON on STDIN. It will be exactly - same as for [system hooks] + same as for [system hooks](../system_hooks/system_hooks.md). That's it! Assuming the file hook code is properly implemented, the hook will fire as appropriate. The file hooks file list is updated for each event, there is no @@ -110,7 +110,3 @@ Validating file hooks from /file_hooks directory * /home/git/gitlab/file_hooks/save_to_file.clj succeed (zero exit code) * /home/git/gitlab/file_hooks/save_to_file.rb failure (non-zero exit code) ``` - -[system hooks]: ../system_hooks/system_hooks.md -[webhooks]: ../user/project/integrations/webhooks.md -[highly available]: ./availability/index.md diff --git a/doc/administration/geo/disaster_recovery/background_verification.md b/doc/administration/geo/disaster_recovery/background_verification.md index 6d6aee08c95..5b24c222f06 100644 --- a/doc/administration/geo/disaster_recovery/background_verification.md +++ b/doc/administration/geo/disaster_recovery/background_verification.md @@ -54,14 +54,14 @@ Feature.enable('geo_repository_verification') Navigate to the **{admin}** **Admin Area >** **{location-dot}** **Geo** dashboard on the **primary** node and expand the **Verification information** tab for that node to view automatic checksumming status for repositories and wikis. Successes are shown in green, pending work -in grey, and failures in red. +in gray, and failures in red.  Navigate to the **{admin}** **Admin Area >** **{location-dot}** **Geo** dashboard on the **secondary** node and expand the **Verification information** tab for that node to view automatic verification status for repositories and wikis. As with checksumming, successes are shown in -green, pending work in grey, and failures in red. +green, pending work in gray, and failures in red.  diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 6f417f955ac..9f7afad353b 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -167,6 +167,45 @@ do this manually. previously for the **secondary**. 1. Success! The **secondary** has now been promoted to **primary**. +#### Promoting a **secondary** node with an external PostgreSQL database + +The `gitlab-ctl promote-to-primary-node` command cannot be used in conjunction with +an external PostgreSQL database, as it can only perform changes on a **secondary** +node with GitLab and the database on the same machine. As a result, a manual process is +required: + +1. Promote the replica database associated with the **secondary** site. This will + set the database to read-write: + - Amazon RDS - [Promoting a Read Replica to Be a Standalone DB Instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Promote) + - Azure Database for PostgreSQL - [Stop replication](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal#stop-replication) + +1. Edit `/etc/gitlab/gitlab.rb` on every node in the **secondary** site to + reflect its new status as **primary** by removing any lines that enabled the + `geo_secondary_role`: + + ```ruby + ## In GitLab 11.4 and earlier, remove this line. + geo_secondary_role['enable'] = true + + ## In GitLab 11.5 and later, remove this line. + roles ['geo_secondary_role'] + ``` + + After making these changes [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + on each node so the changes take effect. + +1. Promote the **secondary** to **primary**. SSH into a single secondary application + node and execute: + + ```shell + sudo gitlab-rake geo:set_secondary_as_primary + ``` + +1. Verify you can connect to the newly promoted **primary** site using the URL used + previously for the **secondary** site. + +Success! The **secondary** site has now been promoted to **primary**. + ### Step 4. (Optional) Updating the primary domain DNS record Updating the DNS records for the primary domain to point to the **secondary** node diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 4b3b464b710..00ca1e4b8b0 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -95,7 +95,7 @@ ensure these processes are close to 100% as possible during active use. Navigate to the **{admin}** **Admin Area >** **{location-dot}** **Geo** dashboard on the **secondary** node to review status. Replicated objects (shown in green) should be close to 100%, and there should be no failures (shown in red). If a large proportion of -objects aren't yet replicated (shown in grey), consider giving the node more +objects aren't yet replicated (shown in gray), consider giving the node more time to complete  diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 0b076e7ff3c..86a8e5b28d1 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -262,7 +262,7 @@ You can login to the **secondary** node with the same credentials you used for t **secondary** Geo node and if Geo is enabled. The initial replication, or 'backfill', will probably still be in progress. You -can monitor the synchronization process on each geo node from the **primary** +can monitor the synchronization process on each Geo node from the **primary** node's **Geo Nodes** dashboard in your browser.  @@ -314,19 +314,17 @@ It is important to note that selective synchronization: Selective synchronization restrictions are implemented on the **secondary** nodes, not the **primary** node. -### Git operations on unreplicated respositories +### Git operations on unreplicated repositories -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2562) in GitLab 12.10. +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2562) in GitLab 12.10 for HTTP(S) and in GitLab 13.0 for SSH. -Git clone, pull, and push operations over HTTP(S) are supported for repositories that +Git clone, pull, and push operations over HTTP(S) and SSH are supported for repositories that exist on the **primary** node but not on **secondary** nodes. This situation can occur when: - Selective synchronization does not include the project attached to the repository. - The repository is actively being replicated but has not completed yet. -SSH [support is planned](https://gitlab.com/groups/gitlab-org/-/epics/2562). - ## Upgrading Geo See the [updating the Geo nodes document](updating_the_geo_nodes.md). diff --git a/doc/administration/geo/replication/database.md b/doc/administration/geo/replication/database.md index ffdec5a83c7..62bd0e6ac19 100644 --- a/doc/administration/geo/replication/database.md +++ b/doc/administration/geo/replication/database.md @@ -33,9 +33,9 @@ recover. See below for more details. The following guide assumes that: -- You are using Omnibus and therefore you are using PostgreSQL 9.6 or later - which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/9.6/app-pgbasebackup.html) and improved - [Foreign Data Wrapper](https://www.postgresql.org/docs/9.6/postgres-fdw.html) support. +- You are using Omnibus and therefore you are using PostgreSQL 11 or later + which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html) and improved + [Foreign Data Wrapper](https://www.postgresql.org/docs/11/postgres-fdw.html) support. - You have a **primary** node already set up (the GitLab server you are replicating from), running Omnibus' PostgreSQL (or equivalent version), and you have a new **secondary** server set up with the same versions of the OS, @@ -91,7 +91,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` postgresql['sql_user_password'] = '<md5_hash_of_your_password>' - # Every node that runs Unicorn or Sidekiq needs to have the database + # Every node that runs Puma or Sidekiq needs to have the database # password specified as below. If you have a high-availability setup, this # must be present in all application nodes. gitlab_rails['db_password'] = '<your_password_here>' @@ -160,7 +160,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o `postgresql['md5_auth_cidr_addresses']` and `postgresql['listen_address']`. The `listen_address` option opens PostgreSQL up to network connections with the interface - corresponding to the given address. See [the PostgreSQL documentation](https://www.postgresql.org/docs/9.6/runtime-config-connection.html) + corresponding to the given address. See [the PostgreSQL documentation](https://www.postgresql.org/docs/11/runtime-config-connection.html) for more details. Depending on your network configuration, the suggested addresses may not @@ -213,7 +213,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` You may also want to edit the `wal_keep_segments` and `max_wal_senders` to match your - database replication requirements. Consult the [PostgreSQL - Replication documentation](https://www.postgresql.org/docs/9.6/runtime-config-replication.html) + database replication requirements. Consult the [PostgreSQL - Replication documentation](https://www.postgresql.org/docs/11/runtime-config-replication.html) for more information. 1. Save the file and reconfigure GitLab for the database listen changes and @@ -273,7 +273,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o 1. Stop application server and Sidekiq ```shell - gitlab-ctl stop unicorn + gitlab-ctl stop puma gitlab-ctl stop sidekiq ``` @@ -442,7 +442,7 @@ data before running `pg_basebackup`. (e.g., you know the network path is secure, or you are using a site-to-site VPN). This is **not** safe over the public Internet! - You can read more details about each `sslmode` in the - [PostgreSQL documentation](https://www.postgresql.org/docs/9.6/libpq-ssl.html#LIBPQ-SSL-PROTECTION); + [PostgreSQL documentation](https://www.postgresql.org/docs/11/libpq-ssl.html#LIBPQ-SSL-PROTECTION); the instructions above are carefully written to ensure protection against both passive eavesdroppers and active "man-in-the-middle" attackers. - Change the `--slot-name` to the name of the replication slot @@ -461,10 +461,10 @@ The replication process is now complete. PostgreSQL connections. We recommend using PgBouncer if you use GitLab in a high-availability configuration with a cluster of nodes supporting a Geo **primary** node and another cluster of nodes supporting a Geo **secondary** node. For more -information, see [High Availability with GitLab Omnibus](../../high_availability/database.md#high-availability-with-gitlab-omnibus-premium-only). +information, see [High Availability with Omnibus GitLab](../../high_availability/database.md#high-availability-with-omnibus-gitlab-premium-only). For a Geo **secondary** node to work properly with PgBouncer in front of the database, -it will need a separate read-only user to make [PostgreSQL FDW queries](https://www.postgresql.org/docs/9.6/postgres-fdw.html) +it will need a separate read-only user to make [PostgreSQL FDW queries](https://www.postgresql.org/docs/11/postgres-fdw.html) work: 1. On the **primary** Geo database, enter the PostgreSQL on the console as an diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 3431df3ed1f..17031b11f51 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -79,7 +79,7 @@ GitLab stores files and blobs such as Issue attachments or LFS objects into eith - A Storage Appliance that exposes an Object Storage-compatible API. When using the filesystem store instead of Object Storage, you need to use network mounted filesystems -to run GitLab when using more than one server (for example with a High Availability setup). +to run GitLab when using more than one server. With respect to replication and verification: @@ -135,6 +135,7 @@ successfully, you must replicate their data using some other means. | CI job artifacts (other than traces) | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/issues/8923) | Verified only manually (*1*) | | Archived traces | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/issues/8923) | Verified only on transfer, or manually (*1*) | | Personal snippets | **Yes** | **Yes** | | +| [Versioned snippets](../../../user/snippets.md#versioned-snippets) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2810) | | | Project snippets | **Yes** | **Yes** | | | Object pools for forked project deduplication | **Yes** | No | | | [Server-side Git Hooks](../../custom_hooks.md) | No | No | | @@ -145,7 +146,7 @@ successfully, you must replicate their data using some other means. | [Maven Repository](../../../user/packages/maven_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | | [Conan Repository](../../../user/packages/conan_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | | [NuGet Repository](../../../user/packages/nuget_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | -| [PyPi Repository](../../../user/packages/pypi_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2554) | No | | +| [PyPi Repository](../../../user/packages/pypi_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2554) | No | | | [External merge request diffs](../../merge_request_diffs.md) | [No](https://gitlab.com/gitlab-org/gitlab/issues/33817) | No | | | Content in object storage | **Yes** | No | | diff --git a/doc/administration/geo/replication/external_database.md b/doc/administration/geo/replication/external_database.md index e305718580e..ae3069a0e40 100644 --- a/doc/administration/geo/replication/external_database.md +++ b/doc/administration/geo/replication/external_database.md @@ -17,6 +17,19 @@ developed and tested. We aim to be compatible with most external sudo -i ``` +1. Edit `/etc/gitlab/gitlab.rb` and add a **unique** ID for your node (arbitrary value): + + ```ruby + # The unique identifier for the Geo node. + gitlab_rails['geo_node_name'] = '<node_name_here>' + ``` + +1. Reconfigure the **primary** node for the change to take effect: + + ```shell + gitlab-ctl reconfigure + ``` + 1. Execute the command below to define the node as **primary** node: ```shell @@ -38,7 +51,14 @@ Given you have a primary node set up on AWS EC2 that uses RDS. You can now just create a read-only replica in a different region and the replication process will be managed by AWS. Make sure you've set Network ACL, Subnet, and Security Group according to your needs, so the secondary application node can access the database. -Skip to the [Configure secondary application node](#configure-secondary-application-nodes-to-use-the-external-read-replica) section below. + +The following instructions detail how to create a read-only replica for common +cloud providers: + +- Amazon RDS - [Creating a Read Replica](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Create) +- Azure Database for PostgreSQL - [Create and manage read replicas in Azure Database for PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal) + +Once your read-only replica is set up, you can skip to [configure you secondary application node](#configure-secondary-application-nodes-to-use-the-external-read-replica). #### Manually configure the primary database for replication @@ -133,6 +153,10 @@ To configure the connection to the external read-replica database and enable Log gitlab_rails['db_username'] = 'gitlab' gitlab_rails['db_host'] = '<database_read_replica_host>' + + # Disable the bundled Omnibus PostgreSQL, since we are + # using an external PostgreSQL + postgresql['enable'] = false ``` 1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) @@ -142,11 +166,17 @@ To configure the connection to the external read-replica database and enable Log **Secondary** nodes use a separate PostgreSQL installation as a tracking database to keep track of replication status and automatically recover from potential replication issues. Omnibus automatically configures a tracking database -when `roles ['geo_secondary_role']` is set. For high availability, -refer to [Geo High Availability](../../availability/index.md). +when `roles ['geo_secondary_role']` is set. If you want to run this database external to Omnibus, please follow the instructions below. -The tracking database requires an [FDW](https://www.postgresql.org/docs/9.6/postgres-fdw.html) +If you are using a cloud-managed service for the tracking database, you may need +to grant additional roles to your tracking database user (by default, this is +`gitlab_geo`): + +- Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. +- Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. + +The tracking database requires an [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) connection with the **secondary** replica database for improved performance. If you have an external database ready to be used as the tracking database, @@ -200,7 +230,7 @@ the tracking database on port 5432. gitlab-rake geo:db:migrate ``` -1. Configure the [PostgreSQL FDW](https://www.postgresql.org/docs/9.6/postgres-fdw.html) +1. Configure the [PostgreSQL FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) connection and credentials: Save the script below in a file, ex. `/tmp/geo_fdw.sh` and modify the connection diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md new file mode 100644 index 00000000000..a8b0bdeb7da --- /dev/null +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -0,0 +1,100 @@ +# Geo validation tests + +The Geo team performs manual testing and validation on common deployment configurations to ensure +that Geo works when upgrading between minor GitLab versions and major PostgreSQL database versions. + +This section contains a journal of recent validation tests and links to the relevant issues. + +## GitLab upgrades + +The following are GitLab upgrade validation tests we performed. + +### February 2020 + +[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/201837): + +- Description: Tested upgrading from GitLab 12.7.5 to the latest GitLab 12.8 package in a multi-server + configuration. +- Outcome: Partial success because we did not run the looping pipeline during the demo to monitor + downtime. + +### January 2020 + +[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/200085): + +- Description: Tested upgrading from GitLab 12.6.x to the latest GitLab 12.7 package in a multi-server + configuration. +- Outcome: Upgrade test was successful. +- Follow up issues: + - [Investigate Geo end-to-end test failures](https://gitlab.com/gitlab-org/gitlab/issues/201823). + - [Add more logging to Geo end-to-end tests](https://gitlab.com/gitlab-org/gitlab/issues/201830). + - [Excess service restarts during zero-downtime upgrade](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5047). + +[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/199836): + +- Description: Tested upgrading from GitLab 12.5.7 to GitLab 12.6.6 in a multi-server configuration. +- Outcome: Upgrade test was successful. +- Follow up issue: + [Update documentation for zero-downtime upgrades to ensure deploy node it not in use](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5046). + +[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/37044): + +- Description: Tested upgrading from GitLab 12.4.x to the latest GitLab 12.5 package in a multi-server + configuration. +- Outcome: Upgrade test was successful. +- Follow up issues: + - [Investigate why HTTP push spec failed on primary node](https://gitlab.com/gitlab-org/gitlab/issues/199825). + - [Investigate if documentation should be modified to include refresh foreign tables task](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5041). + +### October 2019 + +[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/35262): + +- Description: Tested upgrading from GitLab 12.3.5 to GitLab 12.4.1 in a multi-server configuration. +- Outcome: Upgrade test was successful. + +[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/32437): + +- Description: Tested upgrading from GitLab 12.2.8 to GitLab 12.3.5. +- Outcome: Upgrade test was successful. + +[Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/32435): + +- Description: Tested upgrading from GitLab 12.1.9 to GitLab 12.2.8. +- Outcome: Partial success due to possible misconfiguration issues. + +## PostgreSQL upgrades + +The following are PostgreSQL upgrade validation tests we performed. + +### April 2020 + +[PostgreSQL 11 upgrade procedure for Geo installations](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4975): + +- Description: Prior to making PostgreSQL 11 the default version of PostgreSQL in GitLab 12.10, we + tested upgrading to PostgreSQL 11 in Geo deployments on GitLab 12.9. +- Outcome: Partially successful. Issues were discovered in multi-server configurations with a separate + tracking database and concerns were raised about allowing automatic upgrades when Geo enabled. +- Follow up issues: + - [`replicate-geo-database` incorrectly tries to back up repositories](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5241). + - [`pg-upgrade` fails to upgrade a standalone Geo tracking database](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5242). + - [`revert-pg-upgrade` fails to downgrade the PostgreSQL data of a Geo secondary’s standalone tracking database](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5243). + - [Timeout error on Geo secondary read-replica near the end of `gitlab-ctl pg-upgrade`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5235). + +[Verify Geo installation with PostgreSQL 11](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4971): + +- Description: Prior to making PostgreSQL 11 the default version of PostgreSQL in GitLab 12.10, we + tested fresh installations of GitLab 12.9 with Geo installed with PostgreSQL 11. +- Outcome: Installation test was successful. + +### September 2019 + +[Test and validate PostgreSQL 10.0 upgrade for Geo](https://gitlab.com/gitlab-org/gitlab/issues/12092): + +- Description: With the 12.0 release, GitLab required an upgrade to PostgreSQL 10.0. We tested + various upgrade scenarios from GitLab 11.11.5 through to GitLab 12.1.8. +- Outcome: Multiple issues were found when upgrading and addressed in follow-up issues. +- Follow up issues: + - [`gitlab-ctl` reconfigure fails on Redis node in multi-server Geo setup](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4706). + - [Geo multi-server upgrade from 12.0.9 to 12.1.9 does not upgrade PostgreSQL](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4705). + - [Refresh foreign tables fails on app server in multi-server setup after upgrade to 12.1.9](https://gitlab.com/gitlab-org/gitlab/-/issues/32119). diff --git a/doc/administration/geo/replication/high_availability.md b/doc/administration/geo/replication/high_availability.md index 5099e73d5e8..214f15b7565 100644 --- a/doc/administration/geo/replication/high_availability.md +++ b/doc/administration/geo/replication/high_availability.md @@ -1,460 +1,5 @@ -# Geo High Availability **(PREMIUM ONLY)** +--- +redirect_to: 'multiple_servers.md' +--- -This document describes a minimal reference architecture for running Geo -in a high availability configuration. If your HA setup differs from the one -described, it is possible to adapt these instructions to your needs. - -## Architecture overview - - - -_[diagram source - GitLab employees only](https://docs.google.com/drawings/d/1z0VlizKiLNXVVVaERFwgsIOuEgjcUqDTWPdQYsE7Z4c/edit)_ - -The topology above assumes that the **primary** and **secondary** Geo clusters -are located in two separate locations, on their own virtual network -with private IP addresses. The network is configured such that all machines within -one geographic location can communicate with each other using their private IP addresses. -The IP addresses given are examples and may be different depending on the -network topology of your deployment. - -The only external way to access the two Geo deployments is by HTTPS at -`gitlab.us.example.com` and `gitlab.eu.example.com` in the example above. - -NOTE: **Note:** -The **primary** and **secondary** Geo deployments must be able to communicate to each other over HTTPS. - -## Redis and PostgreSQL High Availability - -Geo supports: - -- Redis and PostgreSQL on the **primary** node configured for high availability -- Redis on **secondary** nodes configured for high availability. - -NOTE: **Note:** -Support for PostgreSQL on **secondary** nodes in high availability configuration -[is planned](https://gitlab.com/groups/gitlab-org/-/epics/2536). - -Because of the additional complexity involved in setting up this configuration -for PostgreSQL and Redis, it is not covered by this Geo HA documentation. - -For more information about setting up a highly available PostgreSQL cluster and Redis cluster using the omnibus package see the high availability documentation for -[PostgreSQL](../../high_availability/database.md) and -[Redis](../../high_availability/redis.md), respectively. - -NOTE: **Note:** -It is possible to use cloud hosted services for PostgreSQL and Redis, but this is beyond the scope of this document. - -## Prerequisites: Two working GitLab HA clusters - -One cluster will serve as the **primary** node. Use the -[GitLab HA documentation](../../availability/index.md) to set this up. If -you already have a working GitLab instance that is in-use, it can be used as a -**primary**. - -The second cluster will serve as the **secondary** node. Again, use the -[GitLab HA documentation](../../availability/index.md) to set this up. -It's a good idea to log in and test it, however, note that its data will be -wiped out as part of the process of replicating from the **primary**. - -## Configure the GitLab cluster to be the **primary** node - -The following steps enable a GitLab cluster to serve as the **primary** node. - -### Step 1: Configure the **primary** frontend servers - -1. Edit `/etc/gitlab/gitlab.rb` and add the following: - - ```ruby - ## - ## Enable the Geo primary role - ## - roles ['geo_primary_role'] - - ## - ## The unique identifier for the Geo node. - ## - gitlab_rails['geo_node_name'] = '<node_name_here>' - - ## - ## Disable automatic migrations - ## - gitlab_rails['auto_migrate'] = false - ``` - -After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. - -NOTE: **Note:** PostgreSQL and Redis should have already been disabled on the -application servers, and connections from the application servers to those -services on the backend servers configured, during normal GitLab HA set up. See -high availability configuration documentation for -[PostgreSQL](../../high_availability/database.md#configuring-the-application-nodes) -and [Redis](../../high_availability/redis.md#example-configuration-for-the-gitlab-application). - -### Step 2: Configure the **primary** database - -1. Edit `/etc/gitlab/gitlab.rb` and add the following: - - ```ruby - ## - ## Configure the Geo primary role and the PostgreSQL role - ## - roles ['geo_primary_role', 'postgres_role'] - ``` - -## Configure a **secondary** node - -A **secondary** cluster is similar to any other GitLab HA cluster, with two -major differences: - -- The main PostgreSQL database is a read-only replica of the **primary** node's - PostgreSQL database. -- There is also a single PostgreSQL database for the **secondary** cluster, - called the "tracking database", which tracks the synchronization state of - various resources. - -Therefore, we will set up the HA components one-by-one, and include deviations -from the normal HA setup. However, we highly recommend first configuring a -brand-new cluster as if it were not part of a Geo setup so that it can be -tested and verified as a working cluster. And only then should it be modified -for use as a Geo **secondary**. This helps to separate problems that are related -and are not related to Geo setup. - -### Step 1: Configure the Redis and Gitaly services on the **secondary** node - -Configure the following services, again using the non-Geo high availability -documentation: - -- [Configuring Redis for GitLab HA](../../high_availability/redis.md) for high - availability. -- [Gitaly](../../high_availability/gitaly.md), which will store data that is - synchronized from the **primary** node. - -NOTE: **Note:** -[NFS](../../high_availability/nfs.md) can be used in place of Gitaly but is not -recommended. - -### Step 2: Configure the main read-only replica PostgreSQL database on the **secondary** node - -NOTE: **Note:** The following documentation assumes the database will be run on -a single node only. PostgreSQL HA on **secondary** nodes is -[not currently supported](https://gitlab.com/groups/gitlab-org/-/epics/2536). - -Configure the [**secondary** database](database.md) as a read-only replica of -the **primary** database. Use the following as a guide. - -1. Generate an MD5 hash of the desired password for the database user that the - GitLab application will use to access the read-replica database: - - Note that the username (`gitlab` by default) is incorporated into the hash. - - ```shell - gitlab-ctl pg-password-md5 gitlab - # Enter password: <your_password_here> - # Confirm password: <your_password_here> - # fca0b89a972d69f00eb3ec98a5838484 - ``` - - Use this hash to fill in `<md5_hash_of_your_password>` in the next step. - -1. Edit `/etc/gitlab/gitlab.rb` in the replica database machine, and add the - following: - - ```ruby - ## - ## Configure the Geo secondary role and the PostgreSQL role - ## - roles ['geo_secondary_role', 'postgres_role'] - - ## - ## Secondary address - ## - replace '<secondary_node_ip>' with the public or VPC address of your Geo secondary node - ## - replace '<tracking_database_ip>' with the public or VPC address of your Geo tracking database node - ## - postgresql['listen_address'] = '<secondary_node_ip>' - postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32', '<tracking_database_ip>/32'] - - ## - ## Database credentials password (defined previously in primary node) - ## - replicate same values here as defined in primary node - ## - postgresql['sql_user_password'] = '<md5_hash_of_your_password>' - gitlab_rails['db_password'] = '<your_password_here>' - - ## - ## When running the Geo tracking database on a separate machine, disable it - ## here and allow connections from the tracking database host. And ensure - ## the tracking database IP is in postgresql['md5_auth_cidr_addresses'] above. - ## - geo_postgresql['enable'] = false - - ## - ## Disable `geo_logcursor` service so Rails doesn't get configured here - ## - geo_logcursor['enable'] = false - ``` - -After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. - -If using an external PostgreSQL instance, refer also to -[Geo with external PostgreSQL instances](external_database.md). - -### Step 3: Configure the tracking database on the **secondary** node - -NOTE: **Note:** This documentation assumes the tracking database will be run on -only a single machine, rather than as a PostgreSQL cluster. - -Configure the tracking database. - -1. Generate an MD5 hash of the desired password for the database user that the - GitLab application will use to access the tracking database: - - Note that the username (`gitlab_geo` by default) is incorporated into the - hash. - - ```shell - gitlab-ctl pg-password-md5 gitlab_geo - # Enter password: <your_password_here> - # Confirm password: <your_password_here> - # fca0b89a972d69f00eb3ec98a5838484 - ``` - - Use this hash to fill in `<tracking_database_password_md5_hash>` in the next - step. - -1. Edit `/etc/gitlab/gitlab.rb` in the tracking database machine, and add the - following: - - ```ruby - ## - ## Enable the Geo secondary tracking database - ## - geo_postgresql['enable'] = true - geo_postgresql['listen_address'] = '<ip_address_of_this_host>' - geo_postgresql['sql_user_password'] = '<tracking_database_password_md5_hash>' - - ## - ## Configure FDW connection to the replica database - ## - geo_secondary['db_fdw'] = true - geo_postgresql['fdw_external_password'] = '<replica_database_password_plaintext>' - geo_postgresql['md5_auth_cidr_addresses'] = ['<replica_database_ip>/32'] - gitlab_rails['db_host'] = '<replica_database_ip>' - - # Prevent reconfigure from attempting to run migrations on the replica DB - gitlab_rails['auto_migrate'] = false - - ## - ## Disable all other services that aren't needed, since we don't have a role - ## that does this. - ## - alertmanager['enable'] = false - consul['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - node_exporter['enable'] = false - pgbouncer_exporter['enable'] = false - postgresql['enable'] = false - prometheus['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - repmgr['enable'] = false - sidekiq['enable'] = false - unicorn['enable'] = false - ``` - -After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. - -If using an external PostgreSQL instance, refer also to -[Geo with external PostgreSQL instances](external_database.md). - -### Step 4: Configure the frontend application servers on the **secondary** node - -In the architecture overview, there are two machines running the GitLab -application services. These services are enabled selectively in the -configuration. - -Configure the application servers following -[Configuring GitLab for HA](../../high_availability/gitlab.md), then make the -following modifications: - -1. Edit `/etc/gitlab/gitlab.rb` on each application server in the **secondary** - cluster, and add the following: - - ```ruby - ## - ## Enable the Geo secondary role - ## - roles ['geo_secondary_role', 'application_role'] - - ## - ## The unique identifier for the Geo node. - ## - gitlab_rails['geo_node_name'] = '<node_name_here>' - - ## - ## Disable automatic migrations - ## - gitlab_rails['auto_migrate'] = false - - ## - ## Configure the connection to the tracking DB. And disable application - ## servers from running tracking databases. - ## - geo_secondary['db_host'] = '<geo_tracking_db_host>' - geo_secondary['db_password'] = '<geo_tracking_db_password>' - geo_postgresql['enable'] = false - - ## - ## Configure connection to the streaming replica database, if you haven't - ## already - ## - gitlab_rails['db_host'] = '<replica_database_host>' - gitlab_rails['db_password'] = '<replica_database_password>' - - ## - ## Configure connection to Redis, if you haven't already - ## - gitlab_rails['redis_host'] = '<redis_host>' - gitlab_rails['redis_password'] = '<redis_password>' - - ## - ## If you are using custom users not managed by Omnibus, you need to specify - ## UIDs and GIDs like below, and ensure they match between servers in a - ## cluster to avoid permissions issues - ## - user['uid'] = 9000 - user['gid'] = 9000 - web_server['uid'] = 9001 - web_server['gid'] = 9001 - registry['uid'] = 9002 - registry['gid'] = 9002 - ``` - -NOTE: **Note:** -If you had set up PostgreSQL cluster using the omnibus package and you had set -up `postgresql['sql_user_password'] = 'md5 digest of secret'` setting, keep in -mind that `gitlab_rails['db_password']` and `geo_secondary['db_password']` -mentioned above contains the plaintext passwords. This is used to let the Rails -servers connect to the databases. - -NOTE: **Note:** -Make sure that current node IP is listed in `postgresql['md5_auth_cidr_addresses']` setting of your remote database. - -After making these changes [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. - -On the secondary the following GitLab frontend services will be enabled: - -- `geo-logcursor` -- `gitlab-pages` -- `gitlab-workhorse` -- `logrotate` -- `nginx` -- `registry` -- `remote-syslog` -- `sidekiq` -- `unicorn` - -Verify these services by running `sudo gitlab-ctl status` on the frontend -application servers. - -### Step 5: Set up the LoadBalancer for the **secondary** node - -In this topology, a load balancer is required at each geographic location to -route traffic to the application servers. - -See [Load Balancer for GitLab HA](../../high_availability/load_balancer.md) for -more information. - -### Step 6: Configure the backend application servers on the **secondary** node - -The minimal reference architecture diagram above shows all application services -running together on the same machines. However, for high availability we -[strongly recommend running all services separately](../../availability/index.md). - -For example, a Sidekiq server could be configured similarly to the frontend -application servers above, with some changes to run only the `sidekiq` service: - -1. Edit `/etc/gitlab/gitlab.rb` on each Sidekiq server in the **secondary** - cluster, and add the following: - - ```ruby - ## - ## Enable the Geo secondary role - ## - roles ['geo_secondary_role'] - - ## - ## Enable the Sidekiq service - ## - sidekiq['enable'] = true - - ## - ## Ensure unnecessary services are disabled - ## - alertmanager['enable'] = false - consul['enable'] = false - geo_logcursor['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - node_exporter['enable'] = false - pgbouncer_exporter['enable'] = false - postgresql['enable'] = false - prometheus['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - repmgr['enable'] = false - unicorn['enable'] = false - - ## - ## The unique identifier for the Geo node. - ## - gitlab_rails['geo_node_name'] = '<node_name_here>' - - ## - ## Disable automatic migrations - ## - gitlab_rails['auto_migrate'] = false - - ## - ## Configure the connection to the tracking DB. And disable application - ## servers from running tracking databases. - ## - geo_secondary['db_host'] = '<geo_tracking_db_host>' - geo_secondary['db_password'] = '<geo_tracking_db_password>' - geo_postgresql['enable'] = false - - ## - ## Configure connection to the streaming replica database, if you haven't - ## already - ## - gitlab_rails['db_host'] = '<replica_database_host>' - gitlab_rails['db_password'] = '<replica_database_password>' - - ## - ## Configure connection to Redis, if you haven't already - ## - gitlab_rails['redis_host'] = '<redis_host>' - gitlab_rails['redis_password'] = '<redis_password>' - - ## - ## If you are using custom users not managed by Omnibus, you need to specify - ## UIDs and GIDs like below, and ensure they match between servers in a - ## cluster to avoid permissions issues - ## - user['uid'] = 9000 - user['gid'] = 9000 - web_server['uid'] = 9001 - web_server['gid'] = 9001 - registry['uid'] = 9002 - registry['gid'] = 9002 - ``` - - You can similarly configure a server to run only the `geo-logcursor` service - with `geo_logcursor['enable'] = true` and disabling Sidekiq with - `sidekiq['enable'] = false`. - - These servers do not need to be attached to the load balancer. +This document was moved to [another location](multiple_servers.md). diff --git a/doc/administration/geo/replication/index.md b/doc/administration/geo/replication/index.md index 7c661abef9a..87bd7b69515 100644 --- a/doc/administration/geo/replication/index.md +++ b/doc/administration/geo/replication/index.md @@ -2,7 +2,7 @@ > - Introduced in GitLab Enterprise Edition 8.9. > - Using Geo in combination with -> [High Availability](../../availability/index.md) +> [multi-server architectures](../../reference_architectures/index.md) > is considered **Generally Available** (GA) in > [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. @@ -110,7 +110,7 @@ The following are required to run Geo: The following operating systems are known to ship with a current version of OpenSSH: - [CentOS](https://www.centos.org) 7.4+ - [Ubuntu](https://ubuntu.com) 16.04+ -- PostgreSQL 9.6+ with [FDW](https://www.postgresql.org/docs/9.6/postgres-fdw.html) support and [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) +- PostgreSQL 11+ with [FDW](https://www.postgresql.org/docs/11/postgres-fdw.html) support and [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) - Git 2.9+ - All nodes must run the same GitLab version. @@ -134,7 +134,7 @@ The following table lists basic ports that must be open between the **primary** See the full list of ports used by GitLab in [Package defaults](https://docs.gitlab.com/omnibus/package-information/defaults.html) NOTE: **Note:** -[Web terminal](../../../ci/environments.md#web-terminals) support requires your load balancer to correctly handle WebSocket connections. +[Web terminal](../../../ci/environments/index.md#web-terminals) support requires your load balancer to correctly handle WebSocket connections. When using HTTP or HTTPS proxying, your load balancer must be configured to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the [web terminal](../../integration/terminal.md) integration guide for more details. NOTE: **Note:** @@ -206,9 +206,9 @@ For information on configuring Geo, see [Geo configuration](configuration.md). For information on how to update your Geo nodes to the latest GitLab version, see [Updating the Geo nodes](updating_the_geo_nodes.md). -### Configuring Geo high availability +### Configuring Geo for multiple servers -For information on configuring Geo for high availability, see [Geo High Availability](high_availability.md). +For information on configuring Geo for multiple servers, see [Geo for multiple servers](multiple_servers.md). ### Configuring Geo with Object Storage @@ -245,7 +245,7 @@ This list of limitations only reflects the latest version of GitLab. If you are - Pushing directly to a **secondary** node redirects (for HTTP) or proxies (for SSH) the request to the **primary** node instead of [handling it directly](https://gitlab.com/gitlab-org/gitlab/issues/1381), except when using Git over HTTP with credentials embedded within the URI. For example, `https://user:password@secondary.tld`. - Cloning, pulling, or pushing repositories that exist on the **primary** node but not on the **secondary** nodes where [selective synchronization](configuration.md#selective-synchronization) does not include the project is not supported over SSH [but support is planned](https://gitlab.com/groups/gitlab-org/-/epics/2562). HTTP(S) is supported. -- The **primary** node has to be online for OAuth login to happen. Existing sessions and Git are not affected. +- The **primary** node has to be online for OAuth login to happen. Existing sessions and Git are not affected. Support for the **secondary** node to use an OAuth provider independent from the primary is [being planned](https://gitlab.com/gitlab-org/gitlab/issues/208465). - The installation takes multiple manual steps that together can take about an hour depending on circumstances. We are working on improving this experience. See [Omnibus GitLab issue #2978](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2978) for details. - Real-time updates of issues/merge requests (for example, via long polling) doesn't work on the **secondary** node. - [Selective synchronization](configuration.md#selective-synchronization) applies only to files and repositories. Other datasets are replicated to the **secondary** node in full, making it inappropriate for use as an access control mechanism. diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md new file mode 100644 index 00000000000..9322c4cc417 --- /dev/null +++ b/doc/administration/geo/replication/multiple_servers.md @@ -0,0 +1,459 @@ +# Geo for multiple servers **(PREMIUM ONLY)** + +This document describes a minimal reference architecture for running Geo +in a multi-server configuration. If your multi-server setup differs from the one +described, it is possible to adapt these instructions to your needs. + +## Architecture overview + + + +_[diagram source - GitLab employees only](https://docs.google.com/drawings/d/1z0VlizKiLNXVVVaERFwgsIOuEgjcUqDTWPdQYsE7Z4c/edit)_ + +The topology above assumes that the **primary** and **secondary** Geo clusters +are located in two separate locations, on their own virtual network +with private IP addresses. The network is configured such that all machines within +one geographic location can communicate with each other using their private IP addresses. +The IP addresses given are examples and may be different depending on the +network topology of your deployment. + +The only external way to access the two Geo deployments is by HTTPS at +`gitlab.us.example.com` and `gitlab.eu.example.com` in the example above. + +NOTE: **Note:** +The **primary** and **secondary** Geo deployments must be able to communicate to each other over HTTPS. + +## Redis and PostgreSQL for multiple servers + +Geo supports: + +- Redis and PostgreSQL on the **primary** node configured for multiple servers. +- Redis on **secondary** nodes configured for multiple servers. + +NOTE: **Note:** +Support for PostgreSQL on **secondary** nodes in multi-server configuration +[is planned](https://gitlab.com/groups/gitlab-org/-/epics/2536). + +Because of the additional complexity involved in setting up this configuration +for PostgreSQL and Redis, it is not covered by this Geo multi-server documentation. + +For more information about setting up a multi-server PostgreSQL cluster and Redis cluster using the omnibus package see the multi-server documentation for +[PostgreSQL](../../high_availability/database.md) and +[Redis](../../high_availability/redis.md), respectively. + +NOTE: **Note:** +It is possible to use cloud hosted services for PostgreSQL and Redis, but this is beyond the scope of this document. + +## Prerequisites: Two working GitLab multi-server clusters + +One cluster will serve as the **primary** node. Use the +[GitLab multi-server documentation](../../reference_architectures/index.md) to set this up. If +you already have a working GitLab instance that is in-use, it can be used as a +**primary**. + +The second cluster will serve as the **secondary** node. Again, use the +[GitLab multi-server documentation](../../reference_architectures/index.md) to set this up. +It's a good idea to log in and test it, however, note that its data will be +wiped out as part of the process of replicating from the **primary**. + +## Configure the GitLab cluster to be the **primary** node + +The following steps enable a GitLab cluster to serve as the **primary** node. + +### Step 1: Configure the **primary** frontend servers + +1. Edit `/etc/gitlab/gitlab.rb` and add the following: + + ```ruby + ## + ## Enable the Geo primary role + ## + roles ['geo_primary_role'] + + ## + ## The unique identifier for the Geo node. + ## + gitlab_rails['geo_node_name'] = '<node_name_here>' + + ## + ## Disable automatic migrations + ## + gitlab_rails['auto_migrate'] = false + ``` + +After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. + +NOTE: **Note:** PostgreSQL and Redis should have already been disabled on the +application servers, and connections from the application servers to those +services on the backend servers configured, during normal GitLab multi-server set up. See +multi-server configuration documentation for +[PostgreSQL](../../high_availability/database.md#configuring-the-application-nodes) +and [Redis](../../high_availability/redis.md#example-configuration-for-the-gitlab-application). + +### Step 2: Configure the **primary** database + +1. Edit `/etc/gitlab/gitlab.rb` and add the following: + + ```ruby + ## + ## Configure the Geo primary role and the PostgreSQL role + ## + roles ['geo_primary_role', 'postgres_role'] + ``` + +## Configure a **secondary** node + +A **secondary** cluster is similar to any other GitLab multi-server cluster, with two +major differences: + +- The main PostgreSQL database is a read-only replica of the **primary** node's + PostgreSQL database. +- There is also a single PostgreSQL database for the **secondary** cluster, + called the "tracking database", which tracks the synchronization state of + various resources. + +Therefore, we will set up the multi-server components one-by-one, and include deviations +from the normal multi-server setup. However, we highly recommend first configuring a +brand-new cluster as if it were not part of a Geo setup so that it can be +tested and verified as a working cluster. And only then should it be modified +for use as a Geo **secondary**. This helps to separate problems that are related +and are not related to Geo setup. + +### Step 1: Configure the Redis and Gitaly services on the **secondary** node + +Configure the following services, again using the non-Geo multi-server +documentation: + +- [Configuring Redis for GitLab](../../high_availability/redis.md) for multiple servers. +- [Gitaly](../../high_availability/gitaly.md), which will store data that is + synchronized from the **primary** node. + +NOTE: **Note:** +[NFS](../../high_availability/nfs.md) can be used in place of Gitaly but is not +recommended. + +### Step 2: Configure the main read-only replica PostgreSQL database on the **secondary** node + +NOTE: **Note:** The following documentation assumes the database will be run on +a single node only. Multi-server PostgreSQL on **secondary** nodes is +[not currently supported](https://gitlab.com/groups/gitlab-org/-/epics/2536). + +Configure the [**secondary** database](database.md) as a read-only replica of +the **primary** database. Use the following as a guide. + +1. Generate an MD5 hash of the desired password for the database user that the + GitLab application will use to access the read-replica database: + + Note that the username (`gitlab` by default) is incorporated into the hash. + + ```shell + gitlab-ctl pg-password-md5 gitlab + # Enter password: <your_password_here> + # Confirm password: <your_password_here> + # fca0b89a972d69f00eb3ec98a5838484 + ``` + + Use this hash to fill in `<md5_hash_of_your_password>` in the next step. + +1. Edit `/etc/gitlab/gitlab.rb` in the replica database machine, and add the + following: + + ```ruby + ## + ## Configure the Geo secondary role and the PostgreSQL role + ## + roles ['geo_secondary_role', 'postgres_role'] + + ## + ## Secondary address + ## - replace '<secondary_node_ip>' with the public or VPC address of your Geo secondary node + ## - replace '<tracking_database_ip>' with the public or VPC address of your Geo tracking database node + ## + postgresql['listen_address'] = '<secondary_node_ip>' + postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32', '<tracking_database_ip>/32'] + + ## + ## Database credentials password (defined previously in primary node) + ## - replicate same values here as defined in primary node + ## + postgresql['sql_user_password'] = '<md5_hash_of_your_password>' + gitlab_rails['db_password'] = '<your_password_here>' + + ## + ## When running the Geo tracking database on a separate machine, disable it + ## here and allow connections from the tracking database host. And ensure + ## the tracking database IP is in postgresql['md5_auth_cidr_addresses'] above. + ## + geo_postgresql['enable'] = false + + ## + ## Disable `geo_logcursor` service so Rails doesn't get configured here + ## + geo_logcursor['enable'] = false + ``` + +After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. + +If using an external PostgreSQL instance, refer also to +[Geo with external PostgreSQL instances](external_database.md). + +### Step 3: Configure the tracking database on the **secondary** node + +NOTE: **Note:** This documentation assumes the tracking database will be run on +only a single machine, rather than as a PostgreSQL cluster. + +Configure the tracking database. + +1. Generate an MD5 hash of the desired password for the database user that the + GitLab application will use to access the tracking database: + + Note that the username (`gitlab_geo` by default) is incorporated into the + hash. + + ```shell + gitlab-ctl pg-password-md5 gitlab_geo + # Enter password: <your_password_here> + # Confirm password: <your_password_here> + # fca0b89a972d69f00eb3ec98a5838484 + ``` + + Use this hash to fill in `<tracking_database_password_md5_hash>` in the next + step. + +1. Edit `/etc/gitlab/gitlab.rb` in the tracking database machine, and add the + following: + + ```ruby + ## + ## Enable the Geo secondary tracking database + ## + geo_postgresql['enable'] = true + geo_postgresql['listen_address'] = '<ip_address_of_this_host>' + geo_postgresql['sql_user_password'] = '<tracking_database_password_md5_hash>' + + ## + ## Configure FDW connection to the replica database + ## + geo_secondary['db_fdw'] = true + geo_postgresql['fdw_external_password'] = '<replica_database_password_plaintext>' + geo_postgresql['md5_auth_cidr_addresses'] = ['<replica_database_ip>/32'] + gitlab_rails['db_host'] = '<replica_database_ip>' + + # Prevent reconfigure from attempting to run migrations on the replica DB + gitlab_rails['auto_migrate'] = false + + ## + ## Disable all other services that aren't needed, since we don't have a role + ## that does this. + ## + alertmanager['enable'] = false + consul['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + node_exporter['enable'] = false + pgbouncer_exporter['enable'] = false + postgresql['enable'] = false + prometheus['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + repmgr['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + ``` + +After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. + +If using an external PostgreSQL instance, refer also to +[Geo with external PostgreSQL instances](external_database.md). + +### Step 4: Configure the frontend application servers on the **secondary** node + +In the architecture overview, there are two machines running the GitLab +application services. These services are enabled selectively in the +configuration. + +Configure the application servers following +[Configuring GitLab for multiple servers](../../high_availability/gitlab.md), then make the +following modifications: + +1. Edit `/etc/gitlab/gitlab.rb` on each application server in the **secondary** + cluster, and add the following: + + ```ruby + ## + ## Enable the Geo secondary role + ## + roles ['geo_secondary_role', 'application_role'] + + ## + ## The unique identifier for the Geo node. + ## + gitlab_rails['geo_node_name'] = '<node_name_here>' + + ## + ## Disable automatic migrations + ## + gitlab_rails['auto_migrate'] = false + + ## + ## Configure the connection to the tracking DB. And disable application + ## servers from running tracking databases. + ## + geo_secondary['db_host'] = '<geo_tracking_db_host>' + geo_secondary['db_password'] = '<geo_tracking_db_password>' + geo_postgresql['enable'] = false + + ## + ## Configure connection to the streaming replica database, if you haven't + ## already + ## + gitlab_rails['db_host'] = '<replica_database_host>' + gitlab_rails['db_password'] = '<replica_database_password>' + + ## + ## Configure connection to Redis, if you haven't already + ## + gitlab_rails['redis_host'] = '<redis_host>' + gitlab_rails['redis_password'] = '<redis_password>' + + ## + ## If you are using custom users not managed by Omnibus, you need to specify + ## UIDs and GIDs like below, and ensure they match between servers in a + ## cluster to avoid permissions issues + ## + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` + +NOTE: **Note:** +If you had set up PostgreSQL cluster using the omnibus package and you had set +up `postgresql['sql_user_password'] = 'md5 digest of secret'` setting, keep in +mind that `gitlab_rails['db_password']` and `geo_secondary['db_password']` +mentioned above contains the plaintext passwords. This is used to let the Rails +servers connect to the databases. + +NOTE: **Note:** +Make sure that current node IP is listed in `postgresql['md5_auth_cidr_addresses']` setting of your remote database. + +After making these changes [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. + +On the secondary the following GitLab frontend services will be enabled: + +- `geo-logcursor` +- `gitlab-pages` +- `gitlab-workhorse` +- `logrotate` +- `nginx` +- `registry` +- `remote-syslog` +- `sidekiq` +- `puma` + +Verify these services by running `sudo gitlab-ctl status` on the frontend +application servers. + +### Step 5: Set up the LoadBalancer for the **secondary** node + +In this topology, a load balancer is required at each geographic location to +route traffic to the application servers. + +See [Load Balancer for GitLab with multiple servers](../../high_availability/load_balancer.md) for +more information. + +### Step 6: Configure the backend application servers on the **secondary** node + +The minimal reference architecture diagram above shows all application services +running together on the same machines. However, for multiple servers we +[strongly recommend running all services separately](../../reference_architectures/index.md). + +For example, a Sidekiq server could be configured similarly to the frontend +application servers above, with some changes to run only the `sidekiq` service: + +1. Edit `/etc/gitlab/gitlab.rb` on each Sidekiq server in the **secondary** + cluster, and add the following: + + ```ruby + ## + ## Enable the Geo secondary role + ## + roles ['geo_secondary_role'] + + ## + ## Enable the Sidekiq service + ## + sidekiq['enable'] = true + + ## + ## Ensure unnecessary services are disabled + ## + alertmanager['enable'] = false + consul['enable'] = false + geo_logcursor['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + node_exporter['enable'] = false + pgbouncer_exporter['enable'] = false + postgresql['enable'] = false + prometheus['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + repmgr['enable'] = false + puma['enable'] = false + + ## + ## The unique identifier for the Geo node. + ## + gitlab_rails['geo_node_name'] = '<node_name_here>' + + ## + ## Disable automatic migrations + ## + gitlab_rails['auto_migrate'] = false + + ## + ## Configure the connection to the tracking DB. And disable application + ## servers from running tracking databases. + ## + geo_secondary['db_host'] = '<geo_tracking_db_host>' + geo_secondary['db_password'] = '<geo_tracking_db_password>' + geo_postgresql['enable'] = false + + ## + ## Configure connection to the streaming replica database, if you haven't + ## already + ## + gitlab_rails['db_host'] = '<replica_database_host>' + gitlab_rails['db_password'] = '<replica_database_password>' + + ## + ## Configure connection to Redis, if you haven't already + ## + gitlab_rails['redis_host'] = '<redis_host>' + gitlab_rails['redis_password'] = '<redis_password>' + + ## + ## If you are using custom users not managed by Omnibus, you need to specify + ## UIDs and GIDs like below, and ensure they match between servers in a + ## cluster to avoid permissions issues + ## + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` + + You can similarly configure a server to run only the `geo-logcursor` service + with `geo_logcursor['enable'] = true` and disabling Sidekiq with + `sidekiq['enable'] = false`. + + These servers do not need to be attached to the load balancer. diff --git a/doc/administration/geo/replication/security_review.md b/doc/administration/geo/replication/security_review.md index 18fe1ad22cd..0ac8157220a 100644 --- a/doc/administration/geo/replication/security_review.md +++ b/doc/administration/geo/replication/security_review.md @@ -73,7 +73,7 @@ from [owasp.org](https://owasp.org/). - Nothing Geo-specific. Any user where `admin: true` is set in the database is considered an admin with super-user privileges. - See also: [more granular access control](https://gitlab.com/gitlab-org/gitlab-foss/issues/32730) - (not geo-specific) + (not Geo-specific). - Much of Geo’s integration (database replication, for instance) must be configured with the application, typically by system administrators. @@ -177,7 +177,7 @@ from [owasp.org](https://owasp.org/). ### What databases and application servers support the application? -- PostgreSQL >= 9.6, Redis, Sidekiq, Unicorn. +- PostgreSQL >= 11, Redis, Sidekiq, Puma. ### How will database connection strings, encryption keys, and other sensitive components be stored, accessed, and protected from unauthorized detection? diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index fae9705e935..293414a6e5e 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -497,6 +497,12 @@ to start again from scratch, there are a few steps that can help you: gitlab-ctl start ``` +1. Refresh Foreign Data Wrapper tables + + ```shell + gitlab-rake geo:db:refresh_foreign_tables + ``` + ## Fixing errors during a failover or when promoting a secondary to a primary node The following are possible errors that might be encountered during failover or @@ -538,6 +544,27 @@ or `gitlab-ctl promote-to-primary-node`, either: bug](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22021) was fixed. +### Message: ``NoMethodError: undefined method `secondary?' for nil:NilClass`` + +When [promoting a **secondary** node](../disaster_recovery/index.md#step-3-promoting-a-secondary-node), +you might encounter the following error: + +```plaintext +sudo gitlab-rake geo:set_secondary_as_primary + +rake aborted! +NoMethodError: undefined method `secondary?' for nil:NilClass +/opt/gitlab/embedded/service/gitlab-rails/ee/lib/tasks/geo.rake:232:in `block (3 levels) in <top (required)>' +/opt/gitlab/embedded/service/gitlab-rails/ee/lib/tasks/geo.rake:221:in `block (2 levels) in <top (required)>' +/opt/gitlab/embedded/bin/bundle:23:in `load' +/opt/gitlab/embedded/bin/bundle:23:in `<main>' +Tasks: TOP => geo:set_secondary_as_primary +(See full trace by running task with --trace) +``` + +This command is intended to be executed on a secondary node only, and this error +is displayed if you attempt to run this command on a primary node. + ### Message: `sudo: gitlab-pg-ctl: command not found` When @@ -624,9 +651,9 @@ To check the configuration: ``` This password is normally set on the tracking database during - [Step 3: Configure the tracking database on the secondary node](high_availability.md#step-3-configure-the-tracking-database-on-the-secondary-node), + [Step 3: Configure the tracking database on the secondary node](multiple_servers.md#step-3-configure-the-tracking-database-on-the-secondary-node), and it is set on the app nodes during - [Step 4: Configure the frontend application servers on the secondary node](high_availability.md#step-4-configure-the-frontend-application-servers-on-the-secondary-node). + [Step 4: Configure the frontend application servers on the secondary node](multiple_servers.md#step-4-configure-the-frontend-application-servers-on-the-secondary-node). 1. Check whether any tables are present with the following statement: @@ -833,6 +860,8 @@ which Geo expects to have access to. It usually means, either: - An unsupported replication method was used (for example, logical replication). - The instructions to setup a [Geo database replication](database.md) were not followed correctly. +- Your database connection details are incorrect, that is you have specified the wrong + user in your `/etc/gitlab/gitlab.rb` file. A common source of confusion with **secondary** nodes is that it requires two separate PostgreSQL instances: @@ -854,7 +883,7 @@ Make sure you follow the [Geo database replication](database.md) instructions fo ### Geo database version (...) does not match latest migration (...) -If you are using GitLab Omnibus installation, something might have failed during upgrade. You can: +If you are using Omnibus GitLab installation, something might have failed during upgrade. You can: - Run `sudo gitlab-ctl reconfigure`. - Manually trigger the database migration by running: `sudo gitlab-rake geo:db:migrate` as root on the **secondary** node. diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index df66b1b36ec..fa1576e19eb 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -11,6 +11,7 @@ Updating Geo nodes involves performing: Depending on which version of Geo you are updating to/from, there may be different steps. +- [Updating to GitLab 12.9](version_specific_updates.md#updating-to-gitlab-129) - [Updating to GitLab 12.7](version_specific_updates.md#updating-to-gitlab-127) - [Updating to GitLab 12.2](version_specific_updates.md#updating-to-gitlab-122) - [Updating to GitLab 12.1](version_specific_updates.md#updating-to-gitlab-121) @@ -44,7 +45,7 @@ and all **secondary** nodes: Now that the update process is complete, you may want to check whether everything is working correctly: -1. Run the Geo raketask on all nodes, everything should be green: +1. Run the Geo Rake task on all nodes, everything should be green: ```shell sudo gitlab-rake gitlab:geo:check diff --git a/doc/administration/geo/replication/using_a_geo_server.md b/doc/administration/geo/replication/using_a_geo_server.md index 0f55272f667..2fec2b2b59c 100644 --- a/doc/administration/geo/replication/using_a_geo_server.md +++ b/doc/administration/geo/replication/using_a_geo_server.md @@ -1,4 +1,4 @@ -[//]: # (Please update EE::GitLab::GeoGitAccess::GEO_SERVER_DOCS_URL if this file is moved) +<!-- Please update EE::GitLab::GeoGitAccess::GEO_SERVER_DOCS_URL if this file is moved) --> # Using a Geo Server **(PREMIUM ONLY)** diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md index 81868d19c7f..db8bbddec3b 100644 --- a/doc/administration/geo/replication/version_specific_updates.md +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -30,7 +30,7 @@ GitLab 12.2 includes the following minor PostgreSQL updates: This update will occur even if major PostgreSQL updates are disabled. -Before [refreshing Foreign Data Wrapper during a Geo HA upgrade](https://docs.gitlab.com/omnibus/update/README.html#run-post-deployment-migrations-and-checks), +Before [refreshing Foreign Data Wrapper during a Geo upgrade](https://docs.gitlab.com/omnibus/update/README.html#run-post-deployment-migrations-and-checks), restart the Geo tracking database: ```shell @@ -100,8 +100,8 @@ authentication method. postgresql['sql_user_password'] = '<md5_hash_of_your_password>' # Every node that runs Unicorn or Sidekiq needs to have the database - # password specified as below. If you have a high-availability setup, this - # must be present in all application nodes. + # password specified as below. + # This must be present in all application nodes. gitlab_rails['db_password'] = '<your_password_here>' ``` @@ -125,8 +125,8 @@ authentication method. postgresql['sql_user_password'] = '<md5_hash_of_your_password>' # Every node that runs Unicorn or Sidekiq needs to have the database - # password specified as below. If you have a high-availability setup, this - # must be present in all application nodes. + # password specified as below. + # This must be present in all application nodes. gitlab_rails['db_password'] = '<your_password_here>' # Enable Foreign Data Wrapper diff --git a/doc/administration/git_annex.md b/doc/administration/git_annex.md index 4ac70e7fac2..0d44ed9312c 100644 --- a/doc/administration/git_annex.md +++ b/doc/administration/git_annex.md @@ -5,11 +5,11 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/git_annex.html' # Git annex > **Warning:** GitLab has [completely -removed][deprecate-annex-issue] in GitLab 9.0 (2017/03/22). -Read through the [migration guide from git-annex to Git LFS][guide]. +removed](https://gitlab.com/gitlab-org/gitlab/issues/1648) in GitLab 9.0 (2017/03/22). +Read through the [migration guide from git-annex to Git LFS](../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md). The biggest limitation of Git, compared to some older centralized version -control systems, has been the maximum size of the repositories. +control systems has been the maximum size of the repositories. The general recommendation is to not have Git repositories larger than 1GB to preserve performance. Although GitLab has no limit (some repositories in GitLab @@ -21,10 +21,10 @@ larger organizations. Videos, photos, audio, compiled binaries, and many other types of files are too large. As a workaround, people keep artwork-in-progress in a Dropbox folder and only check in the final result. This results in using outdated files, not -having a complete history and increases the risk of losing work. +having a complete history, and increases the risk of losing work. This problem is solved in GitLab Enterprise Edition by integrating the -[git-annex] application. +[git-annex](https://git-annex.branchable.com/) application. `git-annex` allows managing large binaries with Git without checking the contents into Git. @@ -39,7 +39,7 @@ configuration options required to enable it. ### Requirements -`git-annex` needs to be installed both on the server and the client side. +`git-annex` needs to be installed both on the server and the client-side. For Debian-like systems (for example, Debian and Ubuntu) this can be achieved by running: @@ -64,7 +64,7 @@ The Omnibus package will internally set the correct options in all locations. gitlab_shell['git_annex_enabled'] = true ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Configuration for installations from source @@ -86,7 +86,7 @@ one is located in `config.yml` of GitLab Shell. git_annex_enabled: true ``` -1. Save the files and [restart GitLab][] for the changes to take effect. +1. Save the files and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ## Using GitLab git-annex @@ -186,17 +186,17 @@ access files of projects you have access to (developer, maintainer, or owner rol ## How it works -Internally GitLab uses [GitLab Shell] to handle SSH access and this was a great +Internally GitLab uses [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell) to handle SSH access and this was a great integration point for `git-annex`. There is a setting in GitLab Shell so you can disable GitLab Annex support if you want to. ## Troubleshooting tips -Differences in version of `git-annex` on the GitLab server and on local machines +Differences in the version of `git-annex` on the GitLab server and on local machines can cause `git-annex` to raise unpredicted warnings and errors. -Consult the [Annex upgrade page][annex-upgrade] for more information about +Consult the [Annex upgrade page](https://git-annex.branchable.com/upgrades/) for more information about the differences between versions. You can find out which version is installed on your server by navigating to <https://pkgs.org/download/git-annex> and searching for your distribution. @@ -208,7 +208,7 @@ on how to go around the warnings. This warning can appear on the initial `git annex sync --content` and is caused by differences in `git-annex-shell`. You can read more about it -[in this git-annex issue][issue]. +[in this git-annex issue](https://git-annex.branchable.com/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/). One important thing to note is that despite the warning, the `sync` succeeds and the files are pushed to the GitLab repository. @@ -231,12 +231,3 @@ pull origin ok push origin ``` - -[annex-upgrade]: https://git-annex.branchable.com/upgrades/ -[deprecate-annex-issue]: https://gitlab.com/gitlab-org/gitlab/issues/1648 -[git-annex]: https://git-annex.branchable.com/ "git-annex website" -[gitlab shell]: https://gitlab.com/gitlab-org/gitlab-shell "GitLab Shell repository" -[guide]: ../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md -[issue]: https://git-annex.branchable.com/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/ "git-annex issue" -[reconfigure GitLab]: restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: restart_gitlab.md#installations-from-source diff --git a/doc/administration/gitaly/img/praefect_architecture_v12_10.png b/doc/administration/gitaly/img/praefect_architecture_v12_10.png Binary files differindex 7b8f1138b23..024a12b0a5d 100644 --- a/doc/administration/gitaly/img/praefect_architecture_v12_10.png +++ b/doc/administration/gitaly/img/praefect_architecture_v12_10.png diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 20b9f2104de..14b0a6bd450 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -10,6 +10,11 @@ On this page, *Gitaly server* refers to a standalone node that only runs Gitaly and *Gitaly client* is a GitLab Rails app node that runs all other processes except Gitaly. +CAUTION: **Caution:** +From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, +support for NFS for Git repositories is scheduled to be removed. Upgrade to +[Gitaly Cluster](praefect.md) as soon as possible. + ## Architecture Here's a high-level architecture overview of how Gitaly is used. @@ -69,7 +74,7 @@ The following list depicts what the network architecture of Gitaly is: - A GitLab server can use one or more Gitaly servers. - Gitaly addresses must be specified in such a way that they resolve correctly for ALL Gitaly clients. -- Gitaly clients are: Unicorn, Sidekiq, GitLab Workhorse, +- Gitaly clients are: Puma/Unicorn, Sidekiq, GitLab Workhorse, GitLab Shell, Elasticsearch Indexer, and Gitaly itself. - A Gitaly server must be able to make RPC calls **to itself** via its own `(Gitaly address, Gitaly token)` pair as specified in `/config/gitlab.yml`. @@ -101,13 +106,16 @@ Omnibus GitLab or install it from source: **_do not_** provide the `EXTERNAL_URL=` value. - From source: [Install Gitaly](../../install/installation.md#install-gitaly). -### 2. Client side token configuration +### 2. Authentication -Configure a token on the instance that runs the GitLab Rails application. +Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests +to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. **For Omnibus GitLab** -1. On the client node(s), edit `/etc/gitlab/gitlab.rb`: +To configure the Gitaly token: + +1. On the client server, edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_rails['gitaly_token'] = 'abc123secret' @@ -115,9 +123,42 @@ Configure a token on the instance that runs the GitLab Rails application. 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. On the Gitaly server, edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitaly['auth_token'] = 'abc123secret' + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +There are two ways to configure the GitLab-Shell token: + +1. Copy `/etc/gitlab/gitlab-secrets.json` from the client server to same path on the Gitaly server. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +**OR** + +1. On the client server, edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_shell['secret_token'] = 'shellsecret' + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +1. On the Gitaly server, edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_shell['secret_token'] = 'shellsecret' + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + **For installations from source** -1. On the client node(s), edit `/home/git/gitlab/config/gitlab.yml`: +1. Copy `/home/git/gitlab/.gitlab_shell_secret` from the client server to the same path on the Gitaly +server. +1. On the client server, edit `/home/git/gitlab/config/gitlab.yml`: ```yaml gitlab: @@ -138,12 +179,6 @@ documentation on configuring Gitaly authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configuration/README.md#authentication) . -Gitaly must trigger some callbacks to GitLab via GitLab Shell. As a result, -the GitLab Shell secret must be the same between the other GitLab servers and -the Gitaly server. The easiest way to accomplish this is to copy `/etc/gitlab/gitlab-secrets.json` -from an existing GitLab server to the Gitaly server. Without this shared secret, -Git operations in GitLab will result in an API error. - **For Omnibus GitLab** 1. Edit `/etc/gitlab/gitlab.rb`: @@ -160,17 +195,17 @@ Git operations in GitLab will result in an API error. postgresql['enable'] = false redis['enable'] = false nginx['enable'] = false - unicorn['enable'] = false + puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false grafana['enable'] = false - # If you run a seperate monitoring node you can disable these services + # If you run a separate monitoring node you can disable these services alertmanager['enable'] = false prometheus['enable'] = false - # If you don't run a seperate monitoring node you can - # Enable Prometheus access & disable these extra services + # If you don't run a separate monitoring node you can + # enable Prometheus access & disable these extra services. # This makes Prometheus listen on all interfaces. You must use firewalls to restrict access to this address/port. # prometheus['listen_address'] = '0.0.0.0:9090' # prometheus['monitor_kubernetes'] = false @@ -189,10 +224,6 @@ Git operations in GitLab will result in an API error. # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Authentication token to ensure only authorized servers can communicate with - # Gitaly server - gitaly['auth_token'] = 'abc123secret' - # Make Gitaly accept connections on all network interfaces. You must use # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections @@ -230,6 +261,8 @@ Git operations in GitLab will result in an API error. ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Run `sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml` +to confirm that Gitaly can perform callbacks to the internal API. **For installations from source** @@ -271,7 +304,15 @@ Git operations in GitLab will result in an API error. path = '/srv/gitlab/git-data/repositories' ``` +1. On each Gitaly server, edit `/home/git/gitlab-shell/config.yml`: + + ```yaml + gitlab_url: https://gitlab.example.com + ``` + 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. Run `sudo -u git /home/git/gitlab-shell/bin/check -config /home/git/gitlab-shell/config.yml` +to confirm that Gitaly can perform callbacks to the internal API. ### 4. Converting clients to use the Gitaly server @@ -302,11 +343,10 @@ can read and write to `/mnt/gitlab/storage2`. 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, }) - - gitlab_rails['gitaly_token'] = 'abc123secret' ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the client can connect to Gitaly. 1. Tail the logs to see the requests: ```shell @@ -330,9 +370,6 @@ can read and write to `/mnt/gitlab/storage2`. storage2: gitaly_address: tcp://gitaly2.internal:8075 path: /some/dummy/path - - gitaly: - token: 'abc123secret' ``` NOTE: **Note:** @@ -341,6 +378,8 @@ can read and write to `/mnt/gitlab/storage2`. [this issue](https://gitlab.com/gitlab-org/gitaly/issues/1282) is resolved. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. Run `sudo -u git -H bundle exec rake gitlab:gitaly:check RAILS_ENV=production` to +confirm the client can connect to Gitaly. 1. Tail the logs to see the requests: ```shell @@ -397,9 +436,9 @@ with a Gitaly instance that listens for secure connections you will need to use scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. You will need to bring your own certificates as this isn't provided automatically. -The certificate to be used needs to be installed on all Gitaly nodes, and the -certificate (or CA of certificate) on all -client nodes that communicate with it following the procedure described in +The certificate, or its certificate authority, must be installed on all Gitaly +nodes (including the Gitaly node using the certificate) and on all client nodes +that communicate with it following the procedure described in [GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). NOTE: **Note** @@ -430,17 +469,32 @@ To configure Gitaly with TLS: }) gitlab_rails['gitaly_token'] = 'abc123secret' + gitlab_shell['secret_token'] = 'shellsecret' ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) on client node(s). +1. On the client node(s), copy the cert into the `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + 1. On the Gitaly server, create the `/etc/gitlab/ssl` directory and copy your key and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl sudo chmod 755 /etc/gitlab/ssl sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem ``` +1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when +calling into itself: + + ```shell + sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ``` + 1. On the Gitaly server node(s), edit `/etc/gitlab/gitlab.rb` and add: <!-- @@ -463,6 +517,13 @@ To configure Gitaly with TLS: **For installations from source** +1. On the client node(s), add the cert to the system trusted certs: + + ```shell + sudo cp cert.pem /usr/local/share/ca-certificates/gitaly.crt + sudo update-ca-certificates + ``` + 1. On the client node(s), edit `/home/git/gitlab/config/gitlab.yml` as follows: ```yaml @@ -488,13 +549,32 @@ To configure Gitaly with TLS: data will be stored in this folder. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/issues/1282) is resolved. -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) on client node(s). +1. Save the file and[restart GitLab](../restart_gitlab.md#installations-from-source) +on client node(s). +1. Copy `/home/git/gitlab/.gitlab_shell_secret` from the client server to the same +path on the Gitaly server. +1. On the Gitaly server, create or edit `/etc/default/gitlab` and add: + + ```shell + export SSL_CERT_DIR=/etc/gitlab/ssl + ``` + +1. Save the file. 1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl - sudo chmod 700 /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. On the Gitaly server, add the cert to the system trusted certs so Gitaly will trust it +when calling into itself: + + ```shell + sudo cp cert.pem /usr/local/share/ca-certificates/gitaly.crt + sudo update-ca-certificates ``` 1. On the Gitaly server node(s), edit `/home/git/gitaly/config.toml` and add: @@ -781,7 +861,7 @@ two checks. The result of both of these checks is cached. see if we can access filesystem underneath the Gitaly server directly. If so, use the Rugged patch. -To see if GitLab Rails can access the repo filesystem directly, we use +To see if GitLab Rails can access the repository filesystem directly, we use the following heuristic: - Gitaly ensures that the filesystem has a metadata file in its root diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 19a69dc348c..0ea83f0e090 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -1,35 +1,70 @@ -# Praefect: High Availability +--- +type: reference +--- -NOTE: **Note:** Praefect is an experimental service, and data loss is likely. +# Gitaly Cluster -Praefect is an optional reverse-proxy for [Gitaly](../index.md) to manage a -cluster of Gitaly nodes for high availability. Initially, high availability -be implemented through asynchronous replication. If a Gitaly node becomes -unavailable, it will be possible to fail over to a warm Gitaly replica. +[Gitaly](index.md), the service that provides storage for Git repositories, can +be run in a clustered configuration to increase fault tolerance. In this +configuration, every Git repository is stored on every Gitaly node in the +cluster. Multiple clusters (or shards), can be configured. -The first minimal version will support: +NOTE: **Note:** +Gitaly Clusters can be created using [GitLab Core](https://about.gitlab.com/pricing/#self-managed) +and higher tiers. However, technical support is limited to GitLab Premium and Ultimate customers +only. Not available in GitLab.com. + +Praefect is a router and transaction manager for Gitaly, and a required +component for running a Gitaly Cluster. + + + +Using a Gitaly Cluster increase fault tolerance by: + +- Replicating write operations to warm standby Gitaly nodes. +- Detecting Gitaly node failures. +- Automatically routing Git requests to an available Gitaly node. + +The availability objectives for Gitaly clusters are: + +- **Recovery Point Objective (RPO):** Less than 1 minute. + + Writes are replicated asynchronously. Any writes that have not been replicated + to the newly promoted primary are lost. + + [Strong Consistency](https://gitlab.com/groups/gitlab-org/-/epics/1189) is + planned to improve this to "no loss". + +- **Recovery Time Objective (RTO):** Less than 10 seconds. + + Outages are detected by a health checks run by each Praefect node every + second. Failover requires ten consecutive failed health checks on each + Praefect node. + + [Faster outage detection](https://gitlab.com/gitlab-org/gitaly/-/issues/2608) + is planned to improve this to less than 1 second. + +The current version supports: - Eventual consistency of the secondary replicas. -- Automatic fail over from the primary to the secondary. +- Automatic failover from the primary to the secondary. - Reporting of possible data loss if replication queue is non empty. +- Marking the newly promoted primary read only if possible data loss is + detected. Follow the [HA Gitaly epic](https://gitlab.com/groups/gitlab-org/-/epics/1489) -for updates and roadmap. - -## Requirements for configuring Gitaly for High Availability +for improvements including +[horizontally distributing reads](https://gitlab.com/groups/gitlab-org/-/epics/2013). -NOTE: **Note:** this reference architecture is not highly available because -Praefect is a single point of failure. +## Requirements for configuring a Gitaly Cluster -The minimal [alpha](https://about.gitlab.com/handbook/product/#alpha-beta-ga) -reference architecture additionally requires: +The minimum recommended configuration for a Gitaly Cluster requires: -- 1 Praefect node -- 1 PostgreSQL server (PostgreSQL 9.6 or newer) +- 1 load balancer +- 1 PostgreSQL server (PostgreSQL 11 or newer) +- 3 Praefect nodes - 3 Gitaly nodes (1 primary, 2 secondary) - - See the [design document](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/design_ha.md) for implementation details. @@ -43,6 +78,7 @@ package (highly recommended), follow the steps below: 1. [Configuring the Praefect database](#postgresql) 1. [Configuring the Praefect proxy/router](#praefect) 1. [Configuring each Gitaly node](#gitaly) (once for each Gitaly node) +1. [Configure the load balancer](#load-balancer) 1. [Updating the GitLab server configuration](#gitlab) 1. [Configure Grafana](#grafana) @@ -51,8 +87,8 @@ package (highly recommended), follow the steps below: Before beginning, you should already have a working GitLab instance. [Learn how to install GitLab](https://about.gitlab.com/install/). -Provision a PostgreSQL server (PostgreSQL 9.6 or newer). Configuration through -the GitLab Omnibus distribution is not yet supported. Follow this +Provision a PostgreSQL server (PostgreSQL 11 or newer). Configuration through +the Omnibus GitLab distribution is not yet supported. Follow this [issue](https://gitlab.com/gitlab-org/gitaly/issues/2476) for updates. Prepare all your new nodes by [installing @@ -64,6 +100,7 @@ GitLab](https://about.gitlab.com/install/). You will need the IP/host address for each node. +1. `LOAD_BALANCER_SERVER_ADDRESS`: the IP/hots address of the load balancer 1. `POSTGRESQL_SERVER_ADDRESS`: the IP/host address of the PostgreSQL server 1. `PRAEFECT_HOST`: the IP/host address of the Praefect server 1. `GITALY_HOST`: the IP/host address of each Gitaly server @@ -98,17 +135,19 @@ We will note in the instructions below where these secrets are required. ### PostgreSQL -NOTE: **Note:** don't reuse the GitLab application database for the Praefect -database. +NOTE: **Note:** do not store the GitLab application database and the Praefect +database on the same PostgreSQL server if using +[Geo](../geo/replication/index.md). The replication state is internal to each instance +of GitLab and should not be replicated. To complete this section you will need: - 1 Praefect node -- 1 PostgreSQL server (PostgreSQL 9.6 or newer) +- 1 PostgreSQL server (PostgreSQL 11 or newer) - An SQL user with permissions to create databases During this section, we will configure the PostgreSQL server, from the Praefect -node, using `psql` which is installed by GitLab Omnibus. +node, using `psql` which is installed by Omnibus GitLab. 1. SSH into the **Praefect** node and login as root: @@ -173,7 +212,7 @@ application server, or a Gitaly node. nginx['enable'] = false prometheus['enable'] = false grafana['enable'] = false - unicorn['enable'] = false + puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false gitaly['enable'] = false @@ -189,16 +228,12 @@ application server, or a Gitaly node. 1. Configure **Praefect** to listen on network interfaces by editing `/etc/gitlab/gitlab.rb`: - You will need to replace: - - - `PRAEFECT_HOST` with the IP address or hostname of the Praefect node - ```ruby - praefect['listen_addr'] = 'PRAEFECT_HOST:2305' + praefect['listen_addr'] = '0.0.0.0:2305' # Enable Prometheus metrics access to Praefect. You must use firewalls # to restrict access to this address/port. - praefect['prometheus_listen_addr'] = 'PRAEFECT_HOST:9652' + praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ``` 1. Configure a strong `auth_token` for **Praefect** by editing @@ -245,9 +280,9 @@ application server, or a Gitaly node. 1. Configure the **Praefect** cluster to connect to each Gitaly node in the cluster by editing `/etc/gitlab/gitlab.rb`. - In the example below we have configured one cluster named `praefect`. This - cluster has three Gitaly nodes `gitaly-1`, `gitaly-2`, and `gitaly-3`, which - will be replicas of each other. + In the example below we have configured one virtual storage (or shard) named + `storage-1`. This cluster has three Gitaly nodes `gitaly-1`, `gitaly-2`, and + `gitaly-3`, which will be replicas of each other. Replace `PRAEFECT_INTERNAL_TOKEN` with a strong secret, which will be used by Praefect when communicating with Gitaly nodes in the cluster. This token is @@ -260,13 +295,13 @@ application server, or a Gitaly node. NOTE: **Note:** The `gitaly-1` node is currently denoted the primary. This can be used to manually fail from one node to another. This will be removed - in the future to allow for automatic failover. + in the [future](https://gitlab.com/gitlab-org/gitaly/-/issues/2634). ```ruby # Name of storage hash must match storage name in git_data_dirs on GitLab # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { - 'praefect' => { + 'storage-1' => { 'gitaly-1' => { 'address' => 'tcp://GITALY_HOST:8075', 'token' => 'PRAEFECT_INTERNAL_TOKEN', @@ -284,75 +319,63 @@ application server, or a Gitaly node. } ``` -1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): +1. Enable the database replication queue: - ```shell - gitlab-ctl reconfigure + ```ruby + praefect['postgres_queue_enabled'] = true ``` -1. Verify that Praefect can reach PostgreSQL: + In the next release, database replication queue will be enabled by default. + See [issue #2615](https://gitlab.com/gitlab-org/gitaly/-/issues/2615). - ```shell - sudo -u git /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-ping +1. Enable automatic failover by editing `/etc/gitlab/gitlab.rb`: + + ```ruby + praefect['failover_enabled'] = true + praefect['failover_election_strategy'] = 'sql' ``` - If the check fails, make sure you have followed the steps correctly. If you - edit `/etc/gitlab/gitlab.rb`, remember to run `sudo gitlab-ctl reconfigure` - again before trying the `sql-ping` command. + When automatic failover is enabled, Praefect checks the health of internal + Gitaly nodes. If the primary has a certain amount of health checks fail, it + will promote one of the secondaries to be primary, and demote the primary to + be a secondary. -#### Automatic failover + NOTE: **Note:** Database leader election will be [enabled by default in the + future](https://gitlab.com/gitlab-org/gitaly/-/issues/2682). -When automatic failover is enabled, Praefect will do automatic detection of the health of internal Gitaly nodes. If the -primary has a certain amount of health checks fail, it will decide to promote one of the secondaries to be primary, and -demote the primary to be a secondary. + Caution, **automatic failover** favors availability over consistency and will + cause data loss if changes have not been replicated to the newly elected + primary. In the next release, leader election will [prefer to promote up to + date replicas](https://gitlab.com/gitlab-org/gitaly/-/issues/2642), and it + will be an option to favor consistency by marking [out-of-date repositories + read-only](https://gitlab.com/gitlab-org/gitaly/-/issues/2630). -1. To enable automatic failover, edit `/etc/gitlab/gitlab.rb`: +1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure + Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): - ```ruby - # failover_enabled turns on automatic failover - praefect['failover_enabled'] = true - praefect['virtual_storages'] = { - 'praefect' => { - 'gitaly-1' => { - 'address' => 'tcp://GITALY_HOST:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN', - 'primary' => true - }, - 'gitaly-2' => { - 'address' => 'tcp://GITALY_HOST:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' - }, - 'gitaly-3' => { - 'address' => 'tcp://GITALY_HOST:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' - } - } - } + ```shell + gitlab-ctl reconfigure ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. To ensure that Praefect [has updated its Prometheus listen + address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), [restart + Gitaly](../restart_gitlab.md#omnibus-gitlab-restart): -Below is the picture when Praefect starts up with the config.toml above: + ```shell + gitlab-ctl restart praefect + ``` -```mermaid -graph TD - A[Praefect] -->|Mutator RPC| B(internal_storage_0) - B --> |Replication|C[internal_storage_1] -``` +1. Verify that Praefect can reach PostgreSQL: -Let's say suddenly `internal_storage_0` goes down. Praefect will detect this and -automatically switch over to `internal_storage_1`, and `internal_storage_0` will serve as a secondary: + ```shell + sudo -u git /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-ping + ``` -```mermaid -graph TD - A[Praefect] -->|Mutator RPC| B(internal_storage_1) - B --> |Replication|C[internal_storage_0] -``` + If the check fails, make sure you have followed the steps correctly. If you + edit `/etc/gitlab/gitlab.rb`, remember to run `sudo gitlab-ctl reconfigure` + again before trying the `sql-ping` command. -NOTE: **Note:**: Currently this feature is supported for setups that only have 1 Praefect instance. Praefect instances running, -for example behind a load balancer, `failover_enabled` should be disabled. The reason is The reason is because there -is no coordination that currently happens across different Praefect instances, so there could be a situation where -two Praefect instances think two different Gitaly nodes are the primary. +**The steps above must be completed for each Praefect node!** ### Gitaly @@ -365,7 +388,7 @@ To complete this section you will need: These should be dedicated nodes, do not run other services on these nodes. Every Gitaly server assigned to the Praefect cluster needs to be configured. The -configuration is the same as a normal [standalone Gitaly server](../index.md), +configuration is the same as a normal [standalone Gitaly server](index.md), except: - the storage names are exposed to Praefect, not GitLab @@ -403,7 +426,7 @@ documentation](index.md#3-gitaly-server-configuration). nginx['enable'] = false prometheus['enable'] = false grafana['enable'] = false - unicorn['enable'] = false + puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false prometheus_monitoring['enable'] = false @@ -419,18 +442,14 @@ documentation](index.md#3-gitaly-server-configuration). 1. Configure **Gitaly** to listen on network interfaces by editing `/etc/gitlab/gitlab.rb`: - You will need to replace: - - - `GITALY_HOST` with the IP address or hostname of the Gitaly node - ```ruby # Make Gitaly accept connections on all network interfaces. # Use firewalls to restrict access to this address/port. - gitaly['listen_addr'] = 'GITALY_HOST:8075' + gitaly['listen_addr'] = '0.0.0.0:8075' # Enable Prometheus metrics access to Gitaly. You must use firewalls # to restrict access to this address/port. - gitaly['prometheus_listen_addr'] = 'GITALY_HOST:9236' + gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' ``` 1. Configure a strong `auth_token` for **Gitaly** by editing @@ -445,7 +464,7 @@ documentation](index.md#3-gitaly-server-configuration). 1. Configure the GitLab Shell `secret_token`, and `internal_api_url` which are needed for `git push` operations. - If you have already configured [Gitaly on its own server](../index.md) + If you have already configured [Gitaly on its own server](index.md) ```ruby gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN' @@ -458,12 +477,12 @@ documentation](index.md#3-gitaly-server-configuration). 1. Configure the storage location for Git data by setting `git_data_dirs` in `/etc/gitlab/gitlab.rb`. Each Gitaly node should have a unique storage name - (eg `gitaly-1`). + (such as `gitaly-1`). Instead of configuring `git_data_dirs` uniquely for each Gitaly node, it is often easier to have include the configuration for all Gitaly nodes on every Gitaly node. This is supported because the Praefect `virtual_storages` - configuration maps each storage name (eg `gitaly-1`) to a specific node, and + configuration maps each storage name (such as `gitaly-1`) to a specific node, and requests are routed accordingly. This means every Gitaly node in your fleet can share the same configuration. @@ -484,13 +503,16 @@ documentation](index.md#3-gitaly-server-configuration). }) ``` -1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure Gitaly](../restart_gitlab.md#omnibus-gitlab-reconfigure): +1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure + Gitaly](../restart_gitlab.md#omnibus-gitlab-reconfigure): ```shell gitlab-ctl reconfigure ``` -1. To ensure that Gitaly [has updated its Prometheus listen address](https://gitlab.com/gitlab-org/gitaly/-/issues/2521), [restart Gitaly](../restart_gitlab.md#omnibus-gitlab-restart): +1. To ensure that Gitaly [has updated its Prometheus listen + address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), [restart + Gitaly](../restart_gitlab.md#omnibus-gitlab-restart): ```shell gitlab-ctl restart gitaly @@ -508,37 +530,23 @@ config. sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dial-nodes ``` -1. Enable automatic failover by editing `/etc/gitlab/gitlab.rb`: +### Load Balancer - ```ruby - praefect['failover_enabled'] = true - ``` - - When automatic failover is enabled, Praefect checks the health of internal - Gitaly nodes. If the primary has a certain amount of health checks fail, it - will promote one of the secondaries to be primary, and demote the primary to - be a secondary. +In a highly available Gitaly configuration, a load balancer is needed to route +internal traffic from the GitLab application to the Praefect nodes. The +specifics on which load balancer to use or the exact configuration is beyond the +scope of the GitLab documentation. - Manual failover is possible by updating `praefect['virtual_storages']` and - nominating a new primary node. +We hope that if you’re managing HA systems like GitLab, you have a load balancer +of choice already. Some examples include [HAProxy](https://www.haproxy.org/) +(open-source), [Google Internal Load Balancer](https://cloud.google.com/load-balancing/docs/internal/), +[AWS Elastic Load Balancer](https://aws.amazon.com/elasticloadbalancing/), F5 +Big-IP LTM, and Citrix Net Scaler. This documentation will outline what ports +and protocols you need configure. -1. By default, Praefect will nominate a primary Gitaly node for each - shard and store the state of the primary in local memory. This state - does not persist across restarts and will cause a split brain - if multiple Praefect nodes are used for redundancy. - - To avoid this limitation, enable the SQL election strategy: - - ```ruby - praefect['failover_election_strategy'] = 'sql' - ``` - -1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure - Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): - - ```shell - gitlab-ctl reconfigure - ``` +| LB Port | Backend Port | Protocol | +|---------|--------------|----------| +| 2305 | 2305 | TCP | ### GitLab @@ -555,7 +563,7 @@ Particular attention should be shown to: - the storage name added to `git_data_dirs` in this section must match the storage name under `praefect['virtual_storages']` on the Praefect node. This was set in the [Praefect](#praefect) section of this guide. This document uses - `praefect` as the Praefect storage name. + `storage-1` as the Praefect storage name. 1. SSH into the **GitLab** node and login as root: @@ -563,12 +571,23 @@ Particular attention should be shown to: sudo -i ``` +1. Configure the `external_url` so that files could be served by GitLab + by proper endpoint access by editing `/etc/gitlab/gitlab.rb`: + + You will need to replace `GITLAB_SERVER_URL` with the real external facing + URL on which current GitLab instance is serving: + + ```ruby + external_url 'GITLAB_SERVER_URL' + ``` + 1. Add the Praefect cluster as a storage location by editing `/etc/gitlab/gitlab.rb`. You will need to replace: - - `PRAEFECT_HOST` with the IP address or hostname of the Praefect node + - `LOAD_BALANCER_SERVER_ADDRESS` with the IP address or hostname of the load + balancer. - `GITLAB_HOST` with the IP address or hostname of the GitLab server - `PRAEFECT_EXTERNAL_TOKEN` with the real secret @@ -577,18 +596,18 @@ Particular attention should be shown to: "default" => { "gitaly_address" => "tcp://GITLAB_HOST:8075" }, - "praefect" => { - "gitaly_address" => "tcp://PRAEFECT_HOST:2305", + "storage-1" => { + "gitaly_address" => "tcp://LOAD_BALANCER_SERVER_ADDRESS:2305", "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN' } }) ``` -1. Allow Gitaly to listen on a tcp port by editing +1. Allow Gitaly to listen on a TCP port by editing `/etc/gitlab/gitlab.rb` ```ruby - gitaly['listen_addr'] = 'GITLAB_HOST:8075' + gitaly['listen_addr'] = '0.0.0.0:8075' ``` 1. Configure the `gitlab_shell['secret_token']` so that callbacks from Gitaly @@ -601,16 +620,6 @@ Particular attention should be shown to: gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN' ``` -1. Configure the `external_url` so that files could be served by GitLab - by proper endpoint access by editing `/etc/gitlab/gitlab.rb`: - - You will need to replace `GITLAB_SERVER_URL` with the real external facing URL on which - current GitLab instance is serving: - - ```ruby - external_url 'GITLAB_SERVER_URL' - ``` - 1. Add Prometheus monitoring settings by editing `/etc/gitlab/gitlab.rb`. You will need to replace: @@ -624,7 +633,9 @@ Particular attention should be shown to: 'job_name' => 'praefect', 'static_configs' => [ 'targets' => [ - 'PRAEFECT_HOST:9652' # praefect + 'PRAEFECT_HOST:9652', # praefect-1 + 'PRAEFECT_HOST:9652', # praefect-2 + 'PRAEFECT_HOST:9652', # praefect-3 ] ] }, @@ -647,6 +658,14 @@ Particular attention should be shown to: gitlab-ctl reconfigure ``` +1. To ensure that Gitaly [has updated its Prometheus listen + address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), [restart + Gitaly](../restart_gitlab.md#omnibus-gitlab-restart): + + ```shell + gitlab-ctl restart gitaly + ``` + 1. Verify each `gitlab-shell` on each Gitaly instance can reach GitLab. On each Gitaly instance run: ```shell @@ -725,6 +744,13 @@ Praefect regularly checks the health of each backend Gitaly node. This information can be used to automatically failover to a new primary node if the current primary node is found to be unhealthy. +- **PostgreSQL (recommended):** Enabled by setting + `praefect['failover_election_strategy'] = sql`. This configuration + option will allow multiple Praefect nodes to coordinate via the + PostgreSQL database to elect a primary Gitaly node. This configuration + will cause Praefect nodes to elect a new primary, monitor its health, + and elect a new primary if the current one has not been reachable in + 10 seconds by a majority of the Praefect nodes. - **Manual:** Automatic failover is disabled. The primary node can be reconfigured in `/etc/gitlab/gitlab.rb` on the Praefect node. Modify the `praefect['virtual_storages']` field by moving the `primary = true` to promote @@ -735,13 +761,6 @@ current primary node is found to be unhealthy. checks fail for the current primary backend Gitaly node, and new primary will be elected. **Do not use with multiple Praefect nodes!** Using with multiple Praefect nodes is likely to result in a split brain. -- **PostgreSQL:** Enabled by setting - `praefect['failover_election_strategy'] = sql`. This configuration - option will allow multiple Praefect nodes to coordinate via the - PostgreSQL database to elect a primary Gitaly node. This configuration - will cause Praefect nodes to elect a new primary, monitor its health, - and elect a new primary if the current one has not been reachable in - 10 seconds by a majority of the Praefect nodes. NOTE: **Note:**: Praefect does not yet account for replication lag on the secondaries during the election process, so data loss can occur @@ -753,13 +772,13 @@ strategy in the future. ## Identifying Impact of a Primary Node Failure -When a primary Gitaly node fails, there is a chance of dataloss. Dataloss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The Praefect `dataloss` subcommand helps identify these cases by counting the number of dead replication jobs for each repository within a given timeframe. +When a primary Gitaly node fails, there is a chance of data loss. Data loss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The Praefect `dataloss` sub-command helps identify these cases by counting the number of dead replication jobs for each repository within a given time frame. ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss -from <rfc3339-time> -to <rfc3339-time> ``` -If the timeframe is not specified, dead replication jobs from the last six hours are counted: +If the time frame is not specified, dead replication jobs from the last six hours are counted: ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss @@ -770,19 +789,27 @@ example/repository-2: 4 jobs example/repository-3: 2 jobs ``` -To specify a timeframe in UTC, run: +To specify a time frame in UTC, run: ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss -from 2020-01-02T00:00:00+00:00 -to 2020-01-02T00:02:00+00:00 ``` +### Checking repository checksums + +To check a project's checksums across all nodes, the Praefect replicas Rake task can be used: + +```shell +sudo gitlab-rake "gitlab:praefect:replicas[project_id]" +``` + ## Backend Node Recovery When a Praefect backend node fails and is no longer able to replicate changes, the backend node will start to drift from the primary. If that node eventually recovers, it will need to be reconciled with the current primary. The primary node is considered the single source of truth for the -state of a shard. The Praefect `reconcile` subcommand allows for the manual +state of a shard. The Praefect `reconcile` sub-command allows for the manual reconciliation between a backend node and the current primary. Run the following command on the Praefect server after all placeholders diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index 6b6919247fe..e718d8953ca 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -149,7 +149,7 @@ before and after. If the hit ratio does not improve, the higher limit is probably not making a meaningful difference. Here is an example Prometheus query to see the hit rate: -```text +```plaintext sum(rate(gitaly_catfile_cache_total{type="hit"}[5m])) / sum(rate(gitaly_catfile_cache_total{type=~"(hit)|(miss)"}[5m])) ``` @@ -258,7 +258,7 @@ You can adjust the `concurrency` of each RPC endpoint. | ---- | ---- | -------- | ----------- | | `concurrency` | array | yes | An array of RPC endpoints. | | `rpc` | string | no | The name of the RPC endpoint (`/gitaly.RepositoryService/GarbageCollect`). | -| `max_per_repo` | integer | no | Concurrency per RPC per repo. | +| `max_per_repo` | integer | no | Concurrency per RPC per repository. | Example: diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md index 55ec3b8d6c4..d36b029cbb3 100644 --- a/doc/administration/high_availability/README.md +++ b/doc/administration/high_availability/README.md @@ -1,7 +1,7 @@ --- -type: reference, concepts +redirect_to: ../reference_architectures/index.md --- -# High Availability +# Reference Architectures -This content has been moved to the [availability page](../availability/index.md). +This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md index 6762a81f671..1f22c46a0ad 100644 --- a/doc/administration/high_availability/consul.md +++ b/doc/administration/high_availability/consul.md @@ -4,14 +4,17 @@ type: reference # Working with the bundled Consul service **(PREMIUM ONLY)** -As part of its High Availability stack, GitLab Premium includes a bundled version of [Consul](https://www.consul.io/) that can be managed through `/etc/gitlab/gitlab.rb`. +As part of its High Availability stack, GitLab Premium includes a bundled version of [Consul](https://www.consul.io/) that can be managed through `/etc/gitlab/gitlab.rb`. Consul is a service networking solution. When it comes to [GitLab Architecture](../../development/architecture.md), Consul utilization is supported for configuring: + +1. [Monitoring in Scaled and Highly Available environments](monitoring_node.md) +1. [PostgreSQL High Availability with Omnibus](database.md#high-availability-with-omnibus-gitlab-premium-only) A Consul cluster consists of multiple server agents, as well as client agents that run on other nodes which need to talk to the Consul cluster. ## Prerequisites First, make sure to [download/install](https://about.gitlab.com/install/) -GitLab Omnibus **on each node**. +Omnibus GitLab **on each node**. Choose an installation method, then make sure you complete steps: @@ -107,7 +110,7 @@ For larger clusters, it is possible to restart multiple agents at a time. See th Nodes running GitLab-bundled Consul should be: -- Members of a healthy cluster prior to upgrading the GitLab Omnibus package. +- Members of a healthy cluster prior to upgrading the Omnibus GitLab package. - Upgraded one node at a time. NOTE: **NOTE:** diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index f06870be93c..6f1873af993 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -24,16 +24,16 @@ If you use a cloud-managed service, or provide your own PostgreSQL: ## PostgreSQL in a Scaled and Highly Available Environment -This section is relevant for [Scalable and Highly Available Setups](../scaling/index.md). +This section is relevant for [Scalable and Highly Available Setups](../reference_architectures/index.md). ### Provide your own PostgreSQL instance **(CORE ONLY)** If you want to use your own deployed PostgreSQL instance(s), see [Provide your own PostgreSQL instance](#provide-your-own-postgresql-instance-core-only) -for more details. However, you can use the GitLab Omnibus package to easily +for more details. However, you can use the Omnibus GitLab package to easily deploy the bundled PostgreSQL. -### Standalone PostgreSQL using GitLab Omnibus **(CORE ONLY)** +### Standalone PostgreSQL using Omnibus GitLab **(CORE ONLY)** 1. SSH into the PostgreSQL server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab @@ -92,7 +92,7 @@ deploy the bundled PostgreSQL. Advanced configuration options are supported and can be added if needed. -### High Availability with GitLab Omnibus **(PREMIUM ONLY)** +### High Availability with Omnibus GitLab **(PREMIUM ONLY)** > Important notes: > @@ -125,7 +125,7 @@ otherwise the networks will become a single point of failure. #### Architecture - + Database nodes run two services with PostgreSQL: @@ -271,7 +271,7 @@ Few notes on the service itself: #### Installing Omnibus GitLab First, make sure to [download/install](https://about.gitlab.com/install/) -GitLab Omnibus **on each node**. +Omnibus GitLab **on each node**. Make sure you install the necessary dependencies from step 1, add GitLab package repository from step 2. @@ -969,7 +969,7 @@ repmgr['trust_auth_cidr_addresses'] = %w(192.168.1.44/32 db2.example.com) ##### MD5 Authentication If you are running on an untrusted network, repmgr can use md5 authentication -with a [`.pgpass` file](https://www.postgresql.org/docs/9.6/libpq-pgpass.html) +with a [`.pgpass` file](https://www.postgresql.org/docs/11/libpq-pgpass.html) to authenticate. You can specify by IP address, FQDN, or by subnet, using the same format as in @@ -1091,7 +1091,7 @@ If you're running into an issue with a component not outlined here, be sure to c ## Configure using Omnibus -**Note**: We recommend that you follow the instructions here for a full [PostgreSQL cluster](#high-availability-with-gitlab-omnibus-premium-only). +**Note**: We recommend that you follow the instructions here for a full [PostgreSQL cluster](#high-availability-with-omnibus-gitlab-premium-only). If you are reading this section due to an old bookmark, you can find that old documentation [in the repository](https://gitlab.com/gitlab-org/gitlab/blob/v10.1.4/doc/administration/high_availability/database.md#configure-using-omnibus). Read more on high-availability configuration: diff --git a/doc/administration/high_availability/gitaly.md b/doc/administration/high_availability/gitaly.md index 5d66d3c5c94..2e6bcabeb06 100644 --- a/doc/administration/high_availability/gitaly.md +++ b/doc/administration/high_availability/gitaly.md @@ -4,14 +4,10 @@ type: reference # Configuring Gitaly for Scaled and High Availability -Gitaly does not yet support full high availability. However, Gitaly is quite -stable and is in use on GitLab.com. Scaled and highly available GitLab environments -should consider using Gitaly on a separate node. +A [Gitaly Cluster](../gitaly/praefect.md) can be used to increase the fault +tolerance of Gitaly in high availability configurations. -See the [Gitaly HA Epic](https://gitlab.com/groups/gitlab-org/-/epics/289) to -track plans and progress toward high availability support. - -This document is relevant for [Scalable and Highly Available Setups](../scaling/index.md). +This document is relevant for [scalable and highly available setups](../reference_architectures/index.md). ## Running Gitaly on its own server @@ -19,7 +15,7 @@ See [Running Gitaly on its own server](../gitaly/index.md#running-gitaly-on-its- in Gitaly documentation. Continue configuration of other components by going back to the -[High Availability](../availability/index.md#gitlab-components-and-configuration-instructions) page. +[reference architecture](../reference_architectures/index.md#configure-gitlab-to-scale) page. ## Enable Monitoring diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index c9c425d366b..cdafdbc4954 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -2,7 +2,9 @@ type: reference --- -# Configuring GitLab for Scaling and High Availability +# Configuring GitLab application (Rails) + +This section describes how to configure the GitLab application (Rails) component. NOTE: **Note:** There is some additional configuration near the bottom for additional GitLab application servers. It's important to read and understand @@ -34,7 +36,7 @@ is recommended over [NFS](nfs.md) wherever possible for improved performance. mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data ``` -1. Download/install GitLab Omnibus using **steps 1 and 2** from +1. Download/install Omnibus GitLab using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/install/). Do not complete other steps on the download page. 1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. @@ -79,8 +81,8 @@ is recommended over [NFS](nfs.md) wherever possible for improved performance. NOTE: **Note:** To maintain uniformity of links across HA clusters, the `external_url` on the first application server as well as the additional application - servers should point to the external url that users will use to access GitLab. - In a typical HA setup, this will be the url of the load balancer which will + servers should point to the external URL that users will use to access GitLab. + In a typical HA setup, this will be the URL of the load balancer which will route traffic to all GitLab application servers in the HA cluster. NOTE: **Note:** When you specify `https` in the `external_url`, as in the example @@ -131,6 +133,9 @@ need some extra configuration. 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. +NOTE: **Note:** You will need to restart the GitLab applications nodes after an update has occurred and database +migrations performed. + ## Enable Monitoring > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. @@ -157,7 +162,7 @@ If you enable Monitoring, it must be enabled on **all** GitLab servers. node_exporter['listen_address'] = '0.0.0.0:9100' gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' sidekiq['listen_address'] = "0.0.0.0" - unicorn['listen'] = '0.0.0.0' + puma['listen'] = '0.0.0.0' # Add the monitoring node's IP address to the monitoring whitelist and allow it to # scrape the NGINX metrics. Replace placeholder `monitoring.gitlab.example.com` with @@ -169,9 +174,11 @@ If you enable Monitoring, it must be enabled on **all** GitLab servers. 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. CAUTION: **Warning:** - After changing `unicorn['listen']` in `gitlab.rb`, and running `sudo gitlab-ctl reconfigure`, - it can take an extended period of time for Unicorn to complete reloading after receiving a `HUP`. - For more information, see the [issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4401). + If running Unicorn, after changing `unicorn['listen']` in `gitlab.rb`, and + running `sudo gitlab-ctl reconfigure`, it can take an extended period of time + for Unicorn to complete reloading after receiving a `HUP`. For more + information, see the + [issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4401). ## Troubleshooting diff --git a/doc/administration/high_availability/load_balancer.md b/doc/administration/high_availability/load_balancer.md index 43a7c442d8c..beeb57a0e32 100644 --- a/doc/administration/high_availability/load_balancer.md +++ b/doc/administration/high_availability/load_balancer.md @@ -66,7 +66,7 @@ for details on managing SSL certificates and configuring NGINX. | 443 | 443 | TCP or HTTPS (*1*) (*2*) | | 22 | 22 | TCP | -- (*1*): [Web terminal](../../ci/environments.md#web-terminals) support requires +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support requires your load balancer to correctly handle WebSocket connections. When using HTTP or HTTPS proxying, this means your load balancer must be configured to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index 79e583c5fcb..409a4dfecb9 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -8,10 +8,10 @@ type: reference You can configure a Prometheus node to monitor GitLab. -## Standalone Monitoring node using GitLab Omnibus +## Standalone Monitoring node using Omnibus GitLab -The GitLab Omnibus package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md). -The monitoring node is not highly available. See [Scaling and High Availability](../scaling/index.md) +The Omnibus GitLab package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md). +The monitoring node is not highly available. See [Scaling and High Availability](../reference_architectures/index.md) for an overview of GitLab scaling and high availability options. The steps below are the minimum necessary to configure a Monitoring node running Prometheus and Grafana with @@ -64,7 +64,7 @@ Omnibus: redis['enable'] = false redis_exporter['enable'] = false sidekiq['enable'] = false - unicorn['enable'] = false + puma['enable'] = false node_exporter['enable'] = false gitlab_exporter['enable'] = false ``` diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index 66f2986ab2a..d2b8cf65b35 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -7,14 +7,16 @@ type: reference You can view information and options set for each of the mounted NFS file systems by running `nfsstat -m` and `cat /etc/fstab`. +CAUTION: **Caution:** +From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, +support for NFS for Git repositories is scheduled to be removed. Upgrade to +[Gitaly Cluster](../gitaly/praefect.md) as soon as possible. + NOTE: **Note:** Filesystem performance has a big impact on overall GitLab performance, especially for actions that read or write to Git repositories. See [Filesystem Performance Benchmarking](../operations/filesystem_benchmarking.md) for steps to test filesystem performance. -NOTE: **Note:** [Cloud Object Storage service](object_storage.md) with [Gitaly](gitaly.md) -is recommended over NFS wherever possible for improved performance. - ## NFS Server features ### Required features @@ -73,7 +75,7 @@ To disable NFS server delegation, do the following: #### Important notes The kernel bug may be fixed in -[more recent kernels with this commit](https://github.om/torvalds/linux/commit/95da1b3a5aded124dd1bda1e3cdb876184813140). +[more recent kernels with this commit](https://github.com/torvalds/linux/commit/95da1b3a5aded124dd1bda1e3cdb876184813140). Red Hat Enterprise 7 [shipped a kernel update](https://access.redhat.com/errata/RHSA-2019:2029) on August 6, 2019 that may also have resolved this problem. @@ -184,7 +186,7 @@ The NFS man page states: Read the [Linux man page](https://linux.die.net/man/5/nfs) to understand the difference, and if you do use `soft`, ensure that you've taken steps to mitigate the risks. -If you experience behaviour that might have been caused by +If you experience behavior that might have been caused by writes to disk on the NFS server not occurring, such as commits going missing, use the `hard` option, because (from the man page): @@ -271,7 +273,7 @@ following are the 4 locations need to be shared: Other GitLab directories should not be shared between nodes. They contain node-specific files and GitLab code that does not need to be shared. To ship -logs to a central location consider using remote syslog. GitLab Omnibus packages +logs to a central location consider using remote syslog. Omnibus GitLab packages provide configuration for [UDP log shipping](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only). Having multiple NFS mounts will require manually making sure the data directories diff --git a/doc/administration/high_availability/nfs_host_client_setup.md b/doc/administration/high_availability/nfs_host_client_setup.md index 6823c1d9abe..7519ebf028d 100644 --- a/doc/administration/high_availability/nfs_host_client_setup.md +++ b/doc/administration/high_availability/nfs_host_client_setup.md @@ -36,7 +36,7 @@ apt-get install nfs-kernel-server In this setup we will share the home directory on the host with the client. Edit the exports file as below to share the host's home directory with the client. If you have multiple clients running GitLab you must enter the client IP addresses in line in the `/etc/exports` file. -```text +```plaintext #/etc/exports for one client /home <client-ip-address>(rw,sync,no_root_squash,no_subtree_check) @@ -96,7 +96,7 @@ NFS version 4. NFSv3 also supports locking as long as Linux Kernel 2.6.5+ is use We recommend using version 4 and do not specifically test NFSv3. See [NFS documentation](nfs.md#nfs-client-mount-options) for guidance on mount options. -```text +```plaintext #/etc/fstab 10.0.0.1:/nfs/home /nfs/home nfs4 defaults,hard,vers=4.1,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 ``` @@ -111,7 +111,7 @@ default file locations in `gitlab.rb` on the client allows you to have one main point and have all the required locations as subdirectories to use the NFS mount for `git-data`. -```text +```plaintext git_data_dirs({"default" => {"path" => "/nfs/home/var/opt/gitlab-data/git-data"}}) gitlab_rails['uploads_directory'] = '/nfs/home/var/opt/gitlab-data/uploads' gitlab_rails['shared_path'] = '/nfs/home/var/opt/gitlab-data/shared' diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md index f0e76172a00..4c672f49e26 100644 --- a/doc/administration/high_availability/pgbouncer.md +++ b/doc/administration/high_availability/pgbouncer.md @@ -165,7 +165,7 @@ Refer to your preferred Load Balancer's documentation for further guidance. 1. Run `gitlab-ctl reconfigure` -1. On the node running Unicorn, make sure the following is set in `/etc/gitlab/gitlab.rb` +1. On the node running Puma, make sure the following is set in `/etc/gitlab/gitlab.rb` ```ruby gitlab_rails['db_host'] = 'PGBOUNCER_HOST' @@ -215,7 +215,7 @@ To start a session, run ```shell # gitlab-ctl pgb-console Password for user pgbouncer: -psql (9.6.8, server 1.7.2/bouncer) +psql (11.7, server 1.7.2/bouncer) Type "help" for help. pgbouncer=# diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index 1e19e7e6c01..d52c80aec0d 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -27,24 +27,24 @@ These will be necessary when configuring the GitLab application servers later. ## Redis in a Scaled and Highly Available Environment -This section is relevant for [Scalable and Highly Available Setups](../scaling/index.md). +This section is relevant for [scalable and highly available setups](../reference_architectures/index.md). ### Provide your own Redis instance **(CORE ONLY)** If you want to use your own deployed Redis instance(s), see [Provide your own Redis instance](#provide-your-own-redis-instance-core-only) -for more details. However, you can use the GitLab Omnibus package to easily +for more details. However, you can use the Omnibus GitLab package to easily deploy the bundled Redis. -### Standalone Redis using GitLab Omnibus **(CORE ONLY)** +### Standalone Redis using Omnibus GitLab **(CORE ONLY)** -The GitLab Omnibus package can be used to configure a standalone Redis server. +The Omnibus GitLab package can be used to configure a standalone Redis server. In this configuration Redis is not highly available, and represents a single point of failure. However, in a scaled environment the objective is to allow the environment to handle more users or to increase throughput. Redis itself is generally stable and can handle many requests so it is an acceptable -trade off to have only a single instance. See [High Availability](../availability/index.md) -for an overview of GitLab scaling and high availability options. +trade off to have only a single instance. See the [reference architectures](../reference_architectures/index.md) +page for an overview of GitLab scaling and high availability options. The steps below are the minimum necessary to configure a Redis server with Omnibus: @@ -63,7 +63,7 @@ Omnibus: ## Disable all other services sidekiq['enable'] = false gitlab_workhorse['enable'] = false - unicorn['enable'] = false + puma['enable'] = false postgresql['enable'] = false nginx['enable'] = false prometheus['enable'] = false @@ -89,16 +89,16 @@ Advanced configuration options are supported and can be added if needed. Continue configuration of other components by going back to the -[High Availability](../availability/index.md#gitlab-components-and-configuration-instructions) page. +[reference architectures](../reference_architectures/index.md#configure-gitlab-to-scale) page. -### High Availability with GitLab Omnibus **(PREMIUM ONLY)** +### High Availability with Omnibus GitLab **(PREMIUM ONLY)** > Experimental Redis Sentinel support was [introduced in GitLab 8.11](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1877). Starting with 8.14, Redis Sentinel is no longer experimental. If you've used it with versions `< 8.14` before, please check the updated documentation here. -High Availability with [Redis](https://redis.io/) is possible using a **Master** x **Slave** +High Availability with [Redis](https://redis.io/) is possible using a **Master** x **Replica** topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically start the failover procedure. @@ -130,12 +130,12 @@ make sure you read this Overview section to better understand how the components are tied together. You need at least `3` independent machines: physical, or VMs running into -distinct physical machines. It is essential that all master and slaves Redis +distinct physical machines. It is essential that all master and replica Redis instances run in different machines. If you fail to provision the machines in that specific way, any issue with the shared environment can bring your entire setup down. -It is OK to run a Sentinel alongside of a master or slave Redis instance. +It is OK to run a Sentinel alongside of a master or replica Redis instance. There should be no more than one Sentinel on the same machine though. You also need to take into consideration the underlying network topology, @@ -156,16 +156,16 @@ components below. High Availability with Redis requires a few things: - Multiple Redis instances -- Run Redis in a **Master** x **Slave** topology +- Run Redis in a **Master** x **Replica** topology - Multiple Sentinel instances - Application support and visibility to all Sentinel and Redis instances Redis Sentinel can handle the most important tasks in an HA environment and that's to help keep servers online with minimal to no downtime. Redis Sentinel: -- Monitors **Master** and **Slaves** instances to see if they are available -- Promotes a **Slave** to **Master** when the **Master** fails -- Demotes a **Master** to **Slave** when the failed **Master** comes back online +- Monitors **Master** and **Replicas** instances to see if they are available +- Promotes a **Replica** to **Master** when the **Master** fails +- Demotes a **Master** to **Replica** when the failed **Master** comes back online (to prevent data-partitioning) - Can be queried by the application to always connect to the current **Master** server @@ -185,8 +185,8 @@ For a minimal setup, you will install the Omnibus GitLab package in `3` **independent** machines, both with **Redis** and **Sentinel**: - Redis Master + Sentinel -- Redis Slave + Sentinel -- Redis Slave + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel If you are not sure or don't understand why and where the amount of nodes come from, read [Redis setup overview](#redis-setup-overview) and @@ -197,14 +197,14 @@ the Omnibus GitLab package in `5` **independent** machines, both with **Redis** and **Sentinel**: - Redis Master + Sentinel -- Redis Slave + Sentinel -- Redis Slave + Sentinel -- Redis Slave + Sentinel -- Redis Slave + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel +- Redis Replica + Sentinel ### Redis setup overview -You must have at least `3` Redis servers: `1` Master, `2` Slaves, and they +You must have at least `3` Redis servers: `1` Master, `2` Replicas, and they need to each be on independent machines (see explanation above). You can have additional Redis nodes, that will help survive a situation @@ -221,7 +221,7 @@ nodes to be provisioned. See [Sentinel setup overview](#sentinel-setup-overview) documentation for more information. All Redis nodes should be configured the same way and with similar server specs, as -in a failover situation, any **Slave** can be promoted as the new **Master** by +in a failover situation, any **Replica** can be promoted as the new **Master** by the Sentinel servers. The replication requires authentication, so you need to define a password to @@ -241,9 +241,9 @@ need to be available and reachable, so that they can elect the Sentinel **leader who will take all the decisions to restore the service availability by: - Promoting a new **Master** -- Reconfiguring the other **Slaves** and make them point to the new **Master** +- Reconfiguring the other **Replicas** and make them point to the new **Master** - Announce the new **Master** to every other Sentinel peer -- Reconfigure the old **Master** and demote to **Slave** when it comes back online +- Reconfigure the old **Master** and demote to **Replica** when it comes back online You must have at least `3` Redis Sentinel servers, and they need to be each in an independent machine (that are believed to fail independently), @@ -280,18 +280,18 @@ the official documentation: already tried against the same master by a given Sentinel, is two times the failover timeout. -- The time needed for a slave replicating to a wrong master according +- The time needed for a replica replicating to a wrong master according to a Sentinel current configuration, to be forced to replicate with the right master, is exactly the failover timeout (counting since the moment a Sentinel detected the misconfiguration). - The time needed to cancel a failover that is already in progress but - did not produced any configuration change (SLAVEOF NO ONE yet not - acknowledged by the promoted slave). + did not produced any configuration change (REPLICAOF NO ONE yet not + acknowledged by the promoted replica). -- The maximum time a failover in progress waits for all the slaves to be - reconfigured as slaves of the new master. However even after this time - the slaves will be reconfigured by the Sentinels anyway, but not with +- The maximum time a failover in progress waits for all the replicas to be + reconfigured as replicas of the new master. However even after this time + the replicas will be reconfigured by the Sentinels anyway, but not with the exact parallel-syncs progression as specified. ### Available configuration setups @@ -306,12 +306,12 @@ Pick the one that suits your needs. documentation. - [Omnibus GitLab **Community Edition** (CE) package](https://about.gitlab.com/install/?version=ce): Redis is bundled, so you can use the package with only the Redis service enabled as described in steps - 1 and 2 of this document (works for both master and slave setups). To install + 1 and 2 of this document (works for both master and replica setups). To install and configure Sentinel, jump directly to the Sentinel section in the [Redis HA installation from source](redis_source.md#step-3-configuring-the-redis-sentinel-instances) documentation. - [Omnibus GitLab **Enterprise Edition** (EE) package](https://about.gitlab.com/install/?version=ee): Both Redis and Sentinel are bundled in the package, so you can use the EE package to set up the whole - Redis HA infrastructure (master, slave and Sentinel) which is described in + Redis HA infrastructure (master, replica and Sentinel) which is described in this document. - If you have installed GitLab using the Omnibus GitLab packages (CE or EE), but you want to use your own external Redis server, follow steps 1-3 in the @@ -328,9 +328,9 @@ This is the section where we install and set up the new Redis instances. > - We assume that you have installed GitLab and all HA components from scratch. If you > already have it installed and running, read how to > [switch from a single-machine installation to Redis HA](#switching-from-an-existing-single-machine-installation-to-redis-ha). -> - Redis nodes (both master and slaves) will need the same password defined in +> - Redis nodes (both master and replica) will need the same password defined in > `redis['password']`. At any time during a failover the Sentinels can -> reconfigure a node and change its status from master to slave and vice versa. +> reconfigure a node and change its status from master to replica and vice versa. ### Prerequisites @@ -392,9 +392,9 @@ The prerequisites for a HA Redis setup are the following: > `roles ['redis_sentinel_role', 'redis_master_role']`. Read more about high > availability roles at <https://docs.gitlab.com/omnibus/roles/>. -### Step 2. Configuring the slave Redis instances +### Step 2. Configuring the replica Redis instances -1. SSH into the **slave** Redis server. +1. SSH into the **replica** Redis server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - Make sure you select the correct Omnibus package, with the same version @@ -404,8 +404,8 @@ The prerequisites for a HA Redis setup are the following: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_slave_role' - roles ['redis_slave_role'] + # Specify server role as 'redis_replica_role' + roles ['redis_replica_role'] # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -435,10 +435,10 @@ The prerequisites for a HA Redis setup are the following: ``` 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Go through the steps again for all the other slave nodes. +1. Go through the steps again for all the other replica nodes. > Note: You can specify multiple roles like sentinel and Redis as: -> `roles ['redis_sentinel_role', 'redis_slave_role']`. Read more about high +> `roles ['redis_sentinel_role', 'redis_replica_role']`. Read more about high > availability roles at <https://docs.gitlab.com/omnibus/roles/>. --- @@ -542,18 +542,18 @@ multiple machines with the Sentinel daemon. ## already tried against the same master by a given Sentinel, is two ## times the failover timeout. ## - ## - The time needed for a slave replicating to a wrong master according + ## - The time needed for a replica replicating to a wrong master according ## to a Sentinel current configuration, to be forced to replicate ## with the right master, is exactly the failover timeout (counting since ## the moment a Sentinel detected the misconfiguration). ## ## - The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (SLAVEOF NO ONE yet not - ## acknowledged by the promoted slave). + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). ## - ## - The maximum time a failover in progress waits for all the slaves to be - ## reconfigured as slaves of the new master. However even after this time - ## the slaves will be reconfigured by the Sentinels anyway, but not with + ## - The maximum time a failover in progress waits for all the replica to be + ## reconfigured as replicas of the new master. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with ## the exact parallel-syncs progression as specified. # sentinel['failover_timeout'] = 60000 ``` @@ -612,7 +612,7 @@ replicate from this machine first, before de-activating the Redis instance inside it. Your single-machine install will be the initial **Master**, and the `3` others -should be configured as **Slave** pointing to this machine. +should be configured as **Replica** pointing to this machine. After replication catches up, you will need to stop services in the single-machine install, to rotate the **Master** to one of the new nodes. @@ -627,7 +627,7 @@ redis['enable'] = false If you fail to replicate first, you may loose data (unprocessed background jobs). -## Example of a minimal configuration with 1 master, 2 slaves and 3 Sentinels +## Example of a minimal configuration with 1 master, 2 replicas and 3 Sentinels >**Note:** Redis Sentinel is bundled with Omnibus GitLab Enterprise Edition only. For @@ -649,8 +649,8 @@ discussed in [Redis setup overview](#redis-setup-overview) and Here is a list and description of each **machine** and the assigned **IP**: - `10.0.0.1`: Redis Master + Sentinel 1 -- `10.0.0.2`: Redis Slave 1 + Sentinel 2 -- `10.0.0.3`: Redis Slave 2 + Sentinel 3 +- `10.0.0.2`: Redis Replica 1 + Sentinel 2 +- `10.0.0.3`: Redis Replica 2 + Sentinel 3 - `10.0.0.4`: GitLab application Please note that after the initial configuration, if a failover is initiated @@ -684,12 +684,12 @@ sentinel['quorum'] = 2 [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -### Example configuration for Redis slave 1 and Sentinel 2 +### Example configuration for Redis replica 1 and Sentinel 2 In `/etc/gitlab/gitlab.rb`: ```ruby -roles ['redis_sentinel_role', 'redis_slave_role'] +roles ['redis_sentinel_role', 'redis_replica_role'] redis['bind'] = '10.0.0.2' redis['port'] = 6379 redis['password'] = 'redis-password-goes-here' @@ -706,12 +706,12 @@ sentinel['quorum'] = 2 [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -### Example configuration for Redis slave 2 and Sentinel 3 +### Example configuration for Redis replica 2 and Sentinel 3 In `/etc/gitlab/gitlab.rb`: ```ruby -roles ['redis_sentinel_role', 'redis_slave_role'] +roles ['redis_sentinel_role', 'redis_replica_role'] redis['bind'] = '10.0.0.3' redis['port'] = 6379 redis['password'] = 'redis-password-goes-here' @@ -847,11 +847,11 @@ mailroom['enable'] = false ------- -## Redis master/slave Role +## Redis master/replica Role redis_master_role['enable'] = true # enable only one of them -redis_slave_role['enable'] = true # enable only one of them +redis_replica_role['enable'] = true # enable only one of them -# When Redis Master or Slave role are enabled, the following services are +# When Redis Master or Replica role are enabled, the following services are # enabled/disabled. Note that if Redis and Sentinel roles are combined, both # services will be enabled. @@ -863,7 +863,7 @@ postgresql['enable'] = false gitlab_rails['enable'] = false mailroom['enable'] = false -# For Redis Slave role, also change this setting from default 'true' to 'false': +# For Redis Replica role, also change this setting from default 'true' to 'false': redis['master'] = false ``` @@ -894,13 +894,13 @@ You can check if everything is correct by connecting to each server using ``` When connected to a `master` Redis, you will see the number of connected -`slaves`, and a list of each with connection details: +`replicas`, and a list of each with connection details: ```plaintext # Replication role:master -connected_slaves:1 -slave0:ip=10.133.5.21,port=6379,state=online,offset=208037514,lag=1 +connected_replicas:1 +replica0:ip=10.133.5.21,port=6379,state=online,offset=208037514,lag=1 master_repl_offset:208037658 repl_backlog_active:1 repl_backlog_size:1048576 @@ -908,21 +908,21 @@ repl_backlog_first_byte_offset:206989083 repl_backlog_histlen:1048576 ``` -When it's a `slave`, you will see details of the master connection and if +When it's a `replica`, you will see details of the master connection and if its `up` or `down`: ```plaintext # Replication -role:slave +role:replica master_host:10.133.1.58 master_port:6379 master_link_status:up master_last_io_seconds_ago:1 master_sync_in_progress:0 -slave_repl_offset:208096498 -slave_priority:100 -slave_read_only:1 -connected_slaves:0 +replica_repl_offset:208096498 +replica_priority:100 +replica_read_only:1 +connected_replicas:0 master_repl_offset:0 repl_backlog_active:0 repl_backlog_size:1048576 diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md index 95b1397bab4..97be480bc3b 100644 --- a/doc/administration/high_availability/redis_source.md +++ b/doc/administration/high_availability/redis_source.md @@ -40,7 +40,7 @@ This is the section where we install and set up the new Redis instances. - Since Redis 3.2, you must define a password to receive external connections (`requirepass`). - If you are using Redis with Sentinel, you will also need to define the same - password for the slave password definition (`masterauth`) in the same instance. + password for the replica password definition (`masterauth`) in the same instance. In addition, read the prerequisites as described in the [Omnibus Redis HA document](redis.md#prerequisites) since they provide some @@ -72,9 +72,9 @@ Assuming that the Redis master instance IP is `10.0.0.1`: 1. Restart the Redis service for the changes to take effect. -### Step 2. Configuring the slave Redis instances +### Step 2. Configuring the replica Redis instances -Assuming that the Redis slave instance IP is `10.0.0.2`: +Assuming that the Redis replica instance IP is `10.0.0.2`: 1. [Install Redis](../../install/installation.md#7-redis). 1. Edit `/etc/redis/redis.conf`: @@ -95,12 +95,12 @@ Assuming that the Redis slave instance IP is `10.0.0.2`: requirepass redis-password-goes-here masterauth redis-password-goes-here - ## Define `slaveof` pointing to the Redis master instance with IP and port. - slaveof 10.0.0.1 6379 + ## Define `replicaof` pointing to the Redis master instance with IP and port. + replicaof 10.0.0.1 6379 ``` 1. Restart the Redis service for the changes to take effect. -1. Go through the steps again for all the other slave nodes. +1. Go through the steps again for all the other replica nodes. ### Step 3. Configuring the Redis Sentinel instances @@ -131,7 +131,7 @@ master with IP `10.0.0.1` (some settings might overlap with the master): masterauth redis-password-goes-here ## Define with `sentinel auth-pass` the same shared password you have - ## defined for both Redis master and slaves instances. + ## defined for both Redis master and replicas instances. sentinel auth-pass gitlab-redis redis-password-goes-here ## Define with `sentinel monitor` the IP and port of the Redis @@ -149,18 +149,18 @@ master with IP `10.0.0.1` (some settings might overlap with the master): ## already tried against the same master by a given Sentinel, is two ## times the failover timeout. ## - ## * The time needed for a slave replicating to a wrong master according + ## * The time needed for a replica replicating to a wrong master according ## to a Sentinel current configuration, to be forced to replicate ## with the right master, is exactly the failover timeout (counting since ## the moment a Sentinel detected the misconfiguration). ## ## * The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (SLAVEOF NO ONE yet not - ## acknowledged by the promoted slave). + ## did not produced any configuration change (REPLICAOF NO ONE yet not + ## acknowledged by the promoted replica). ## - ## * The maximum time a failover in progress waits for all the slaves to be - ## reconfigured as slaves of the new master. However even after this time - ## the slaves will be reconfigured by the Sentinels anyway, but not with + ## * The maximum time a failover in progress waits for all the replicas to be + ## reconfigured as replicas of the new master. However even after this time + ## the replicas will be reconfigured by the Sentinels anyway, but not with ## the exact parallel-syncs progression as specified. sentinel failover_timeout 30000 ``` @@ -203,7 +203,7 @@ setup: 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. -## Example of minimal configuration with 1 master, 2 slaves and 3 Sentinels +## Example of minimal configuration with 1 master, 2 replicas and 3 Sentinels In this example we consider that all servers have an internal network interface with IPs in the `10.0.0.x` range, and that they can connect @@ -215,13 +215,13 @@ outside ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353 For this example, **Sentinel 1** will be configured in the same machine as the **Redis Master**, **Sentinel 2** and **Sentinel 3** in the same machines as the -**Slave 1** and **Slave 2** respectively. +**Replica 1** and **Replica 2** respectively. Here is a list and description of each **machine** and the assigned **IP**: - `10.0.0.1`: Redis Master + Sentinel 1 -- `10.0.0.2`: Redis Slave 1 + Sentinel 2 -- `10.0.0.3`: Redis Slave 2 + Sentinel 3 +- `10.0.0.2`: Redis Replica 1 + Sentinel 2 +- `10.0.0.3`: Redis Replica 2 + Sentinel 3 - `10.0.0.4`: GitLab application Please note that after the initial configuration, if a failover is initiated @@ -257,7 +257,7 @@ or a failover promotes a different **Master** node. 1. Restart the Redis service for the changes to take effect. -### Example configuration for Redis slave 1 and Sentinel 2 +### Example configuration for Redis replica 1 and Sentinel 2 1. In `/etc/redis/redis.conf`: @@ -266,7 +266,7 @@ or a failover promotes a different **Master** node. port 6379 requirepass redis-password-goes-here masterauth redis-password-goes-here - slaveof 10.0.0.1 6379 + replicaof 10.0.0.1 6379 ``` 1. In `/etc/redis/sentinel.conf`: @@ -282,7 +282,7 @@ or a failover promotes a different **Master** node. 1. Restart the Redis service for the changes to take effect. -### Example configuration for Redis slave 2 and Sentinel 3 +### Example configuration for Redis replica 2 and Sentinel 3 1. In `/etc/redis/redis.conf`: @@ -291,7 +291,7 @@ or a failover promotes a different **Master** node. port 6379 requirepass redis-password-goes-here masterauth redis-password-goes-here - slaveof 10.0.0.1 6379 + replicaof 10.0.0.1 6379 ``` 1. In `/etc/redis/sentinel.conf`: diff --git a/doc/administration/high_availability/sidekiq.md b/doc/administration/high_availability/sidekiq.md index ed273c3b113..493929dcf3b 100644 --- a/doc/administration/high_availability/sidekiq.md +++ b/doc/administration/high_availability/sidekiq.md @@ -88,12 +88,15 @@ you want using steps 1 and 2 from the GitLab downloads page. postgresql['enable'] = false redis['enable'] = false redis_exporter['enable'] = false - unicorn['enable'] = false + puma['enable'] = false gitlab_exporter['enable'] = false ``` 1. Run `gitlab-ctl reconfigure`. +NOTE: **Note:** You will need to restart the Sidekiq nodes after an update has occurred and database +migrations performed. + ## Example configuration Here's what the ending `/etc/gitlab/gitlab.rb` would look like: @@ -116,7 +119,7 @@ postgres_exporter['enable'] = false postgresql['enable'] = false redis['enable'] = false redis_exporter['enable'] = false -unicorn['enable'] = false +puma['enable'] = false gitlab_exporter['enable'] = false ######################################## diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index dcc590bea9c..fcd69464b13 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -182,6 +182,9 @@ gitlab_rails['incoming_email_start_tls'] = false gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The IDLE command timeout. gitlab_rails['incoming_email_idle_timeout'] = 60 + +# Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery +gitlab_rails['incoming_email_expunge_deleted'] = true ``` Example for source installs: @@ -214,12 +217,18 @@ incoming_email: mailbox: "inbox" # The IDLE command timeout. idle_timeout: 60 + + # Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery + expunge_deleted: true ``` #### Gmail Example configuration for Gmail/G Suite. Assumes mailbox `gitlab-incoming@gmail.com`. +NOTE: **Note:** +`incoming_email_email` cannot be a Gmail alias account. + Example for Omnibus installs: ```ruby @@ -249,6 +258,9 @@ gitlab_rails['incoming_email_start_tls'] = false gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The IDLE command timeout. gitlab_rails['incoming_email_idle_timeout'] = 60 + +# Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery +gitlab_rails['incoming_email_expunge_deleted'] = true ``` Example for source installs: @@ -281,6 +293,9 @@ incoming_email: mailbox: "inbox" # The IDLE command timeout. idle_timeout: 60 + + # Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery + expunge_deleted: true ``` #### Microsoft Exchange Server diff --git a/doc/administration/index.md b/doc/administration/index.md index 1f4e23fce8a..fb827ae8573 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -34,8 +34,8 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Install](../install/README.md): Requirements, directory structures, and installation methods. - [Database load balancing](database_load_balancing.md): Distribute database queries among multiple database servers. **(STARTER ONLY)** - [Omnibus support for log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only) **(STARTER ONLY)** -- [High Availability](availability/index.md): Configure multiple servers for scaling or high availability. - - [Installing GitLab HA on Amazon Web Services (AWS)](../install/aws/index.md): Set up GitLab High Availability on Amazon AWS. +- [Reference architectures](reference_architectures/index.md): Add additional resources to support more users. + - [Installing GitLab on Amazon Web Services (AWS)](../install/aws/index.md): Set up GitLab on Amazon AWS. - [Geo](geo/replication/index.md): Replicate your GitLab instance to other geographic locations as a read-only fully operational version. **(PREMIUM ONLY)** - [Disaster Recovery](geo/disaster_recovery/index.md): Quickly fail-over to a different site with minimal effort in a disaster situation. **(PREMIUM ONLY)** - [Pivotal Tile](../install/pivotal/index.md): Deploy GitLab as a preconfigured appliance using Ops Manager (BOSH) for Pivotal Cloud Foundry. **(PREMIUM ONLY)** @@ -64,7 +64,8 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [External Classification Policy Authorization](../user/admin_area/settings/external_authorization.md) **(PREMIUM ONLY)** - [Upload a license](../user/admin_area/license.md): Upload a license to unlock features that are in paid tiers of GitLab. **(STARTER ONLY)** - [Admin Area](../user/admin_area/index.md): for self-managed instance-wide configuration and maintenance. -- [S/MIME Signing](smime_signing_email.md): how to sign all outgoing notification emails with S/MIME +- [S/MIME Signing](smime_signing_email.md): how to sign all outgoing notification emails with S/MIME. +- [Enabling and disabling features flags](feature_flags.md): how to enable and disable GitLab features deployed behind feature flags. #### Customizing GitLab's appearance @@ -76,9 +77,9 @@ Learn how to install, configure, update, and maintain your GitLab instance. ### Maintaining GitLab -- [Raketasks](../raketasks/README.md): Perform various tasks for maintenance, backups, automatic webhooks setup, and more. +- [Rake tasks](../raketasks/README.md): Perform various tasks for maintenance, backups, automatic webhooks setup, and more. - [Backup and restore](../raketasks/backup_restore.md): Backup and restore your GitLab instance. -- [Operations](operations/index.md): Keeping GitLab up and running (clean up Redis sessions, moving repositories, Sidekiq MemoryKiller, Unicorn). +- [Operations](operations/index.md): Keeping GitLab up and running (clean up Redis sessions, moving repositories, Sidekiq MemoryKiller, Puma). - [Restart GitLab](restart_gitlab.md): Learn how to restart GitLab and its components. - [Invalidate Markdown cache](invalidate_markdown_cache.md): Invalidate any cached Markdown. @@ -98,8 +99,8 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Mattermost](https://docs.gitlab.com/omnibus/gitlab-mattermost/): Integrate with [Mattermost](https://mattermost.com), an open source, private cloud workplace for web messaging. - [PlantUML](integration/plantuml.md): Create simple diagrams in AsciiDoc and Markdown documents - created in snippets, wikis, and repos. -- [Web terminals](integration/terminal.md): Provide terminal access to your applications deployed to Kubernetes from within GitLab's CI/CD [environments](../ci/environments.md#web-terminals). + created in snippets, wikis, and repositories. +- [Web terminals](integration/terminal.md): Provide terminal access to your applications deployed to Kubernetes from within GitLab's CI/CD [environments](../ci/environments/index.md#web-terminals). ## User settings and permissions @@ -183,8 +184,6 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [GitLab Performance Monitoring](monitoring/performance/index.md): - [Enable Performance Monitoring](monitoring/performance/gitlab_configuration.md): Enable GitLab Performance Monitoring. - - [GitLab performance monitoring with InfluxDB](monitoring/performance/influxdb_configuration.md): Configure GitLab and InfluxDB for measuring performance metrics. - - [InfluxDB Schema](monitoring/performance/influxdb_schema.md): Measurements stored in InfluxDB. - [GitLab performance monitoring with Prometheus](monitoring/prometheus/index.md): Configure GitLab and Prometheus for measuring performance metrics. - [GitLab performance monitoring with Grafana](monitoring/performance/grafana_configuration.md): Configure GitLab to visualize time series metrics through graphs and dashboards. - [Request Profiling](monitoring/performance/request_profiling.md): Get a detailed profile on slow requests. diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 57acdec4ea2..d0e8f079cb2 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -99,6 +99,29 @@ header. Such emails don't create comments on issues or merge requests. Sentry payloads sent to GitLab have a 1 MB maximum limit, both for security reasons and to limit memory consumption. +## Max offset allowed via REST API for offset-based pagination + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34565) in GitLab 13.0. + +When using offset-based pagination in the REST API, there is a limit to the maximum +requested offset into the set of results. This limit is only applied to endpoints that +support keyset-based pagination. More information about pagination options can be +found in the [API docs section on pagination](../api/README.md#pagination). + +To set this limit on a self-managed installation, run the following in the +[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): + +```ruby +# If limits don't exist for the default plan, you can create one with: +# Plan.default.create_limits! + +Plan.default.limits.update!(offset_pagination_limit: 10000) +``` + +- **Default offset pagination limit:** 50000 + +NOTE: **Note:** Set the limit to `0` to disable it. + ## CI/CD limits ### Number of jobs in active pipelines @@ -180,7 +203,7 @@ Plan.default.limits.update!(ci_pipeline_schedules: 100) ### Incident Management inbound alert limits -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14932) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17859) in GitLab 12.5. Limiting inbound alerts for an incident reduces the number of alerts (issues) that can be created within a period of time, which can help prevent overloading @@ -192,10 +215,16 @@ alerts in the following ways: ### Prometheus Alert JSON payloads -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14929) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19940) in GitLab 12.6. Prometheus alert payloads sent to the `notify.json` endpoint are limited to 1 MB in size. +### Generic Alert JSON payloads + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16441) in GitLab 12.4. + +Alert payloads sent to the `notify.json` endpoint are limited to 1 MB in size. + ## Environment data on Deploy Boards [Deploy Boards](../user/project/deploy_boards.md) load information from Kubernetes about @@ -233,6 +262,10 @@ NOTE: **Note:** Set the limit to `0` to disable it. - [Length restrictions for file and directory names](../user/project/wiki/index.md#length-restrictions-for-file-and-directory-names). +## Snippets limits + +See the [documentation on Snippets settings](snippets/index.md). + ## Push Event Limits ### Webhooks and Project Services diff --git a/doc/administration/instance_review.md b/doc/administration/instance_review.md index bb64c7967b7..8ea4bff252e 100644 --- a/doc/administration/instance_review.md +++ b/doc/administration/instance_review.md @@ -1,6 +1,6 @@ # Instance Review **(CORE ONLY)** -> [Introduced][6995] in [GitLab Core][ee] 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6995) in [GitLab Core](https://about.gitlab.com/pricing/) 11.3. If you are running a medium size instance of GitLab Core edition you are qualified for a free Instance Review. You can find the button in the User menu. @@ -11,6 +11,3 @@ When you click the button you will be redirected to a form with prefilled data o Once you submit the data to GitLab Inc. you can see the initial report. Additionally you will be contacted by our team for further review which should help you to improve your usage of GitLab. - -[6995]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6995 -[ee]: https://about.gitlab.com/pricing/ diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 4af787fd19f..3372d05bd6b 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -4,7 +4,7 @@ When [PlantUML](https://plantuml.com) integration is enabled and configured in GitLab we are able to create simple diagrams in AsciiDoc and Markdown documents -created in snippets, wikis, and repos. +created in snippets, wikis, and repositories. ## PlantUML Server @@ -70,7 +70,7 @@ sudo service tomcat8 restart Once the Tomcat service restarts the PlantUML service will be ready and listening for requests on port 8080: -```text +```plaintext http://localhost:8080/plantuml ``` @@ -115,10 +115,23 @@ that, login with an Admin account and do following: - Check **Enable PlantUML** checkbox. - Set the PlantUML instance as `https://gitlab.example.com/-/plantuml/`. +NOTE: **Note:** If you are using a PlantUML server running v1.2020.9 and +above (for example, [plantuml.com](https://plantuml.com)), set the `PLANTUML_ENCODING` +environment variable to enable the `deflate` compression. On Omnibus, +this can be done set in `/etc/gitlab.rb`: + +```ruby +gitlab_rails['env'] = { 'PLANTUML_ENCODING' => 'deflate' } +``` + +From GitLab 13.1 and later, PlantUML integration now +[requires a header prefix in the URL](https://github.com/plantuml/plantuml/issues/117#issuecomment-6235450160) +to distinguish different encoding types. + ## Creating Diagrams With PlantUML integration enabled and configured, we can start adding diagrams to -our AsciiDoc snippets, wikis and repos using delimited blocks: +our AsciiDoc snippets, wikis, and repositories using delimited blocks: - **Markdown** @@ -131,7 +144,7 @@ our AsciiDoc snippets, wikis and repos using delimited blocks: - **AsciiDoc** - ```text + ```plaintext [plantuml, format="png", id="myDiagram", width="200px"] ---- Bob->Alice : hello @@ -141,7 +154,7 @@ our AsciiDoc snippets, wikis and repos using delimited blocks: - **reStructuredText** - ```text + ```plaintext .. plantuml:: :caption: Caption with **bold** and *italic* @@ -169,10 +182,10 @@ diagram delimiters `@startuml`/`@enduml` as these are replaced by the AsciiDoc ` Some parameters can be added to the AsciiDoc block definition: -- *format*: Can be either `png` or `svg`. Note that `svg` is not supported by +- `format`: Can be either `png` or `svg`. Note that `svg` is not supported by all browsers so use with care. The default is `png`. -- *id*: A CSS id added to the diagram HTML tag. -- *width*: Width attribute added to the image tag. -- *height*: Height attribute added to the image tag. +- `id`: A CSS ID added to the diagram HTML tag. +- `width`: Width attribute added to the image tag. +- `height`: Height attribute added to the image tag. Markdown does not support any parameters and will always use PNG format. diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index 251e4ded8b4..bc5816ce120 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -8,7 +8,7 @@ Only project maintainers and owners can access web terminals. With the introduction of the [Kubernetes integration](../../user/project/clusters/index.md), GitLab gained the ability to store and use credentials for a Kubernetes cluster. One of the things it uses these credentials for is providing access to -[web terminals](../../ci/environments.md#web-terminals) for environments. +[web terminals](../../ci/environments/index.md#web-terminals) for environments. ## How it works diff --git a/doc/administration/issue_closing_pattern.md b/doc/administration/issue_closing_pattern.md index 7b815143597..579b957eb47 100644 --- a/doc/administration/issue_closing_pattern.md +++ b/doc/administration/issue_closing_pattern.md @@ -28,7 +28,7 @@ Because Rubular doesn't understand `%{issue_ref}`, you can replace this by expression of your liking: ```ruby - gitlab_rails['gitlab_issue_closing_pattern'] = "\b((?:[Cc]los(?:e[sd]|ing)|\b[Ff]ix(?:e[sd]|ing)?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?))+)" + gitlab_rails['gitlab_issue_closing_pattern'] = "\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+)" ``` 1. [Reconfigure](restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -39,7 +39,7 @@ Because Rubular doesn't understand `%{issue_ref}`, you can replace this by 1. Change the value of `issue_closing_pattern`: ```yaml - issue_closing_pattern: "\b((?:[Cc]los(?:e[sd]|ing)|\b[Ff]ix(?:e[sd]|ing)?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?))+)" + issue_closing_pattern: "\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+)" ``` 1. [Restart](restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 3777f551996..fbfad46ef65 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -378,7 +378,7 @@ and [projects APIs](../api/projects.md). When GitLab receives an artifacts archive, an archive metadata file is also generated by [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse). This metadata file describes all the entries that are located in the artifacts archive itself. -The metadata file is in a binary format, with additional GZIP compression. +The metadata file is in a binary format, with additional Gzip compression. GitLab does not extract the artifacts archive in order to save space, memory and disk I/O. It instead inspects the metadata file which contains all the @@ -450,13 +450,13 @@ If you need to manually remove job artifacts associated with multiple jobs while ```ruby project = Project.find_by_full_path('path/to/project') - builds_with_artifacts = project.builds.with_artifacts_archive + builds_with_artifacts = project.builds.with_downloadable_artifacts ``` To select all jobs with artifacts across the entire GitLab instance: ```ruby - builds_with_artifacts = Ci::Build.with_artifacts_archive + builds_with_artifacts = Ci::Build.with_downloadable_artifacts ``` 1. Delete job artifacts older than a specific date: diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index 6020d1d2850..5c03ff1f4b2 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -12,8 +12,8 @@ In the following table you can see the phases a log goes through: | Phase | State | Condition | Data flow | Stored path | | -------------- | ------------ | ----------------------- | -----------------------------------------| ----------- | -| 1: patching | log | When a job is running | GitLab Runner => Unicorn => file storage | `#{ROOT_PATH}/gitlab-ci/builds/#{YYYY_mm}/#{project_id}/#{job_id}.log` | -| 2: overwriting | log | When a job is finished | GitLab Runner => Unicorn => file storage | `#{ROOT_PATH}/gitlab-ci/builds/#{YYYY_mm}/#{project_id}/#{job_id}.log` | +| 1: patching | log | When a job is running | GitLab Runner => Puma => file storage | `#{ROOT_PATH}/gitlab-ci/builds/#{YYYY_mm}/#{project_id}/#{job_id}.log` | +| 2: overwriting | log | When a job is finished | GitLab Runner => Puma => file storage | `#{ROOT_PATH}/gitlab-ci/builds/#{YYYY_mm}/#{project_id}/#{job_id}.log` | | 3: archiving | archived log | After a job is finished | Sidekiq moves log to artifacts folder | `#{ROOT_PATH}/gitlab-rails/shared/artifacts/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | | 4: uploading | archived log | After a log is archived | Sidekiq moves archived log to [object storage](#uploading-logs-to-object-storage) (if configured) | `#{bucket_name}/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 71c1ae22305..e2b982448ef 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -2,7 +2,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.html' --- -# GitLab Git LFS Administration +# GitLab Git Large File Storage (LFS) Administration Documentation on how to use Git LFS are under [Managing large binary files with Git LFS doc](../../topics/git/lfs/index.md). @@ -50,7 +50,7 @@ In `config/gitlab.yml`: ## Storing LFS objects in remote object storage -> [Introduced][ee-2760] in [GitLab Premium][eep] 10.0. Brought to GitLab Core in 10.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2760) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. Brought to GitLab Core in 10.7. It is possible to store LFS objects in remote object storage which allows you to offload local hard disk R/W operations, and free up disk space significantly. @@ -129,7 +129,7 @@ Here is a configuration example with Rackspace Cloud Files. NOTE: **Note:** Regardless of whether the container has public access enabled or disabled, Fog will use the TempURL method to grant access to LFS objects. If you see errors in logs referencing -instantiating storage with a temp-url-key, ensure that you have set the key properly +instantiating storage with a `temp-url-key`, ensure that you have set the key properly on the Rackspace API and in `gitlab.rb`. You can verify the value of the key Rackspace has set by sending a GET request with token header to the service access endpoint URL and comparing the output of the returned headers. @@ -141,18 +141,23 @@ There are two ways to manually do the same thing as automatic uploading (describ **Option 1: Rake task** ```shell -rake gitlab:lfs:migrate +gitlab-rake gitlab:lfs:migrate ``` -**Option 2: rails console** +**Option 2: Rails console** + +Log into the Rails console: ```shell -$ sudo gitlab-rails console # Login to rails console +sudo gitlab-rails console +``` -> # Upload LFS files manually -> LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| -> lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? -> end +Upload LFS files manually + +```ruby +LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| + lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? +end ``` ### S3 for Omnibus installations @@ -177,7 +182,7 @@ On Omnibus installations, the settings are prefixed by `lfs_object_store_`: } ``` -1. Save the file and [reconfigure GitLab]s for the changes to take effect. +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Migrate any existing local LFS objects to the object storage: ```shell @@ -213,7 +218,7 @@ For source installations the settings are nested under `lfs:` and then path_style: true ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. 1. Migrate any existing local LFS objects to the object storage: ```shell @@ -244,19 +249,29 @@ If LFS integration is configured with Google Cloud Storage and background upload Sidekiq workers may encounter this error. This is because the uploading timed out with very large files. LFS files up to 6Gb can be uploaded without any extra steps, otherwise you need to use the following workaround. +Log into Rails console: + ```shell -$ sudo gitlab-rails console # Login to rails console - -> # Set up timeouts. 20 minutes is enough to upload 30GB LFS files. -> # These settings are only in effect for the same session, i.e. they are not effective for sidekiq workers. -> ::Google::Apis::ClientOptions.default.open_timeout_sec = 1200 -> ::Google::Apis::ClientOptions.default.read_timeout_sec = 1200 -> ::Google::Apis::ClientOptions.default.send_timeout_sec = 1200 - -> # Upload LFS files manually. This process does not use sidekiq at all. -> LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| -> lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? -> end +sudo gitlab-rails console +``` + +Set up timeouts: + +- These settings are only in effect for the same session. For example, they are not effective for Sidekiq workers. +- 20 minutes (1200 sec) is enough to upload 30GB LFS files: + +```ruby +::Google::Apis::ClientOptions.default.open_timeout_sec = 1200 +::Google::Apis::ClientOptions.default.read_timeout_sec = 1200 +::Google::Apis::ClientOptions.default.send_timeout_sec = 1200 +``` + +Upload LFS files manually (this process does not use Sidekiq at all): + +```ruby +LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| + lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? +end ``` See more information in [!19581](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19581) @@ -268,8 +283,3 @@ See more information in [!19581](https://gitlab.com/gitlab-org/gitlab-foss/-/mer - Only compatible with the Git LFS client versions 1.1.0 and up, or 1.0.2. - The storage statistics currently count each LFS object multiple times for every project linking to it. - -[reconfigure gitlab]: ../restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" -[restart gitlab]: ../restart_gitlab.md#installations-from-source "How to restart GitLab" -[eep]: https://about.gitlab.com/pricing/ "GitLab Premium" -[ee-2760]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2760 diff --git a/doc/administration/libravatar.md b/doc/administration/libravatar.md index c28e701dc25..e1c38b3409f 100644 --- a/doc/administration/libravatar.md +++ b/doc/administration/libravatar.md @@ -19,7 +19,7 @@ the configuration options as follows: ### For HTTP -```yml +```yaml gravatar: enabled: true # gravatar URLs: possible placeholders: %{hash} %{size} %{email} %{username} @@ -28,7 +28,7 @@ the configuration options as follows: ### For HTTPS -```yml +```yaml gravatar: enabled: true # gravatar URLs: possible placeholders: %{hash} %{size} %{email} %{username} diff --git a/doc/administration/logs.md b/doc/administration/logs.md index e4a3514a539..57b12897979 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Log system GitLab has an advanced log system where everything is logged, so you @@ -333,6 +339,9 @@ only. For example: ## `audit_json.log` +NOTE: **Note:** +Most log entries only exist in [GitLab Starter](https://about.gitlab.com/pricing), however a few exist in GitLab Core. + This file lives in `/var/log/gitlab/gitlab-rails/audit_json.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/audit_json.log` for installations from source. @@ -356,7 +365,11 @@ Changes to group or project settings are logged to this file. For example: } ``` -## `sidekiq.log` +## Sidekiq Logs + +For Omnibus installations, some Sidekiq logs reside in `/var/log/gitlab/sidekiq/current` and as follows. + +### `sidekiq.log` This file lives in `/var/log/gitlab/gitlab-rails/sidekiq.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/sidekiq.log` for @@ -413,7 +426,7 @@ For source installations, edit the `gitlab.yml` and set the Sidekiq log_format: json ``` -## `sidekiq_client.log` +### `sidekiq_client.log` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26586) in GitLab 12.9. @@ -424,21 +437,91 @@ installations from source. This file contains logging information about jobs before they are start being processed by Sidekiq, for example before being enqueued. -This logfile follows the same structure as +This log file follows the same structure as [`sidekiq.log`](#sidekiqlog), so it will be structured as JSON if you've configured this for Sidekiq as mentioned above. ## `gitlab-shell.log` -This file lives in `/var/log/gitlab/gitaly/gitlab-shell.log` for -Omnibus GitLab packages or in `/home/git/gitaly/gitlab-shell.log` for -installations from source. +GitLab Shell is used by GitLab for executing Git commands and provide SSH access to Git repositories. + +### For GitLab versions 12.10 and up + +For GitLab version 12.10 and later, there are 2 `gitlab-shell.log` files. Information containing `git-{upload-pack,receive-pack}` requests lives in `/var/log/gitlab/gitlab-shell/gitlab-shell.log`. Information about hooks to GitLab Shell from Gitaly lives in `/var/log/gitlab/gitaly/gitlab-shell.log`. + +Example log entries for `/var/log/gitlab/gitlab-shell/gitlab-shell.log`: + +```json +{ + "duration_ms": 74.104, + "level": "info", + "method": "POST", + "msg": "Finished HTTP request", + "time": "2020-04-17T20:28:46Z", + "url": "http://127.0.0.1:8080/api/v4/internal/allowed" +} +{ + "command": "git-upload-pack", + "git_protocol": "", + "gl_project_path": "root/example", + "gl_repository": "project-1", + "level": "info", + "msg": "executing git command", + "time": "2020-04-17T20:28:46Z", + "user_id": "user-1", + "username": "root" +} +``` + +Example log entries for `/var/log/gitlab/gitaly/gitlab-shell.log`: + +```json +{ + "method": "POST", + "url": "http://127.0.0.1:8080/api/v4/internal/allowed", + "duration": 0.058012959, + "gitaly_embedded": true, + "pid": 16636, + "level": "info", + "msg": "finished HTTP request", + "time": "2020-04-17T20:29:08+00:00" +} +{ + "method": "POST", + "url": "http://127.0.0.1:8080/api/v4/internal/pre_receive", + "duration": 0.031022552, + "gitaly_embedded": true, + "pid": 16636, + "level": "info", + "msg": "finished HTTP request", + "time": "2020-04-17T20:29:08+00:00" +} +``` + +### For GitLab versions 12.5 through 12.9 + +For GitLab 12.5 to 12.9, this file lives in `/var/log/gitlab/gitaly/gitlab-shell.log` for Omnibus GitLab packages or in `/home/git/gitaly/gitlab-shell.log` for installations from source. + +Example log entries: + +```json +{ + "method": "POST", + "url": "http://127.0.0.1:8080/api/v4/internal/post_receive", + "duration": 0.031809164, + "gitaly_embedded": true, + "pid": 27056, + "level": "info", + "msg": "finished HTTP request", + "time": "2020-04-17T16:24:38+00:00" +} +``` + +### For GitLab 12.5 and earlier -NOTE: **Note:** For GitLab 12.5 and earlier, the file lives in `/var/log/gitlab/gitlab-shell/gitlab-shell.log`. -GitLab Shell is used by GitLab for executing Git commands and provide -SSH access to Git repositories. For example: +Example log entries: ```plaintext I, [2015-02-13T06:17:00.671315 #9291] INFO -- : Adding project root/example.git at </var/opt/gitlab/git-data/repositories/root/dcdcdcdcd.git>. @@ -447,15 +530,29 @@ I, [2015-02-13T06:17:00.679433 #9291] INFO -- : Moving existing hooks directory User clone/fetch activity using SSH transport appears in this log as `executing git command <gitaly-upload-pack...`. -## `current` +## Gitaly Logs -This file lives in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). `runit` is packaged with Omnibus and a brief explanation of its purpose is available [in the omnibus documentation](https://docs.gitlab.com/omnibus/architecture/#runit). [Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in unix timestamp format and `gzip`-compressed (e.g. `@1584057562.s`). +This file lives in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). `runit` is packaged with Omnibus and a brief explanation of its purpose is available [in the omnibus documentation](https://docs.gitlab.com/omnibus/architecture/#runit). [Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in Unix timestamp format and `gzip`-compressed (e.g. `@1584057562.s`). -## `unicorn_stderr.log` +### `grpc.log` -This file lives in `/var/log/gitlab/unicorn/unicorn_stderr.log` for -Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stderr.log` for -installations from source. +This file lives in `/var/log/gitlab/gitlab-rails/grpc.log` for Omnibus GitLab packages. Native [gRPC](https://grpc.io/) logging used by Gitaly. + +## `puma_stderr.log` & `puma_stdout.log` + +This file lives in `/var/log/gitlab/puma/puma_stderr.log` and `/var/log/gitlab/puma/puma_stdout.log` for +Omnibus GitLab packages or in `/home/git/gitlab/log/puma_stderr.log` and `/home/git/gitlab/log/puma_stdout.log` +for installations from source. + +## `unicorn_stderr.log` & `unicorn_stdout.log` + +NOTE: **Note:** +Starting with GitLab 13.0, Puma is the default web server used in GitLab +all-in-one package based installations as well as GitLab Helm chart deployments. + +This file lives in `/var/log/gitlab/unicorn/unicorn_stderr.log` and `/var/log/gitlab/unicorn/unicorn_stdout.log` for +Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stderr.log` and `/home/git/gitlab/log/unicorn_stdout.log` +for installations from source. Unicorn is a high-performance forking Web server which is used for serving the GitLab application. You can look at this log if, for @@ -484,7 +581,7 @@ This file lives in `/var/log/gitlab/gitlab-rails/repocheck.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/repocheck.log` for installations from source. -It logs information whenever a [repository check is run][repocheck] on a project. +It logs information whenever a [repository check is run](repository_checks.md) on a project. ## `importer.log` @@ -494,6 +591,8 @@ This file lives in `/var/log/gitlab/gitlab-rails/importer.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/importer.log` for installations from source. +It logs the progress of the import process. + ## `auth.log` > Introduced in GitLab 12.0. @@ -504,9 +603,9 @@ installations from source. This log records: -- Information whenever [Rack Attack] registers an abusive request. -- Requests over the [Rate Limit] on raw endpoints. -- [Protected paths] abusive requests. +- Information whenever [Rack Attack](../security/rack_attack.md) registers an abusive request. +- Requests over the [Rate Limit](../user/admin_area/settings/rate_limits_on_raw_endpoints.md) on raw endpoints. +- [Protected paths](../user/admin_area/settings/protected_paths.md) abusive requests. NOTE: **Note:** From [%12.1](https://gitlab.com/gitlab-org/gitlab-foss/issues/62756), user ID and username are also @@ -523,7 +622,7 @@ installations from source. GraphQL queries are recorded in that file. For example: ```json -{"query_string":"query IntrospectionQuery{__schema {queryType { name },mutationType { name }}}...(etc)","variables":{"a":1,"b":2},"complexity":181,"depth":1,"duration":7} +{"query_string":"query IntrospectionQuery{__schema {queryType { name },mutationType { name }}}...(etc)","variables":{"a":1,"b":2},"complexity":181,"depth":1,"duration_s":7} ``` ## `migrations.log` @@ -538,7 +637,7 @@ installations from source. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19186) in GitLab 12.6. -This file lives in `/var/log/gitlab/mail_room/mail_room_json.log` for +This file lives in `/var/log/gitlab/mailroom/mail_room_json.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/mail_room_json.log` for installations from source. @@ -562,7 +661,7 @@ will be generated in `/var/log/gitlab/gitlab-rails/sidekiq_exporter.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/sidekiq_exporter.log` for installations from source. -If Prometheus metrics and the Web Exporter are both enabled, Unicorn/Puma will +If Prometheus metrics and the Web Exporter are both enabled, Puma/Unicorn will start a Web server and listen to the defined port (default: `8083`). Access logs will be generated in `/var/log/gitlab/gitlab-rails/web_exporter.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/web_exporter.log` for @@ -578,7 +677,7 @@ It's stored at: - `/var/log/gitlab/gitlab-rails/database_load_balancing.log` for Omnibus GitLab packages. - `/home/git/gitlab/log/database_load_balancing.log` for installations from source. -## `elasticsearch.log` +## `elasticsearch.log` **(STARTER ONLY)** > Introduced in GitLab 12.6. @@ -648,7 +747,24 @@ Each line contains a JSON line that can be ingested by Elasticsearch. For exampl } ``` -## `geo.log` +## `service_measurement.log` + +> Introduced in GitLab 13.0. + +This file lives in `/var/log/gitlab/gitlab-rails/service_measurement.log` for +Omnibus GitLab packages or in `/home/git/gitlab/log/service_measurement.log` for +installations from source. + +It contain only a single structured log with measurements for each service execution. +It will contain measurement such as: number of sql calls, execution_time, gc_stats, memory usage, etc... + +For example: + +```json +{ "severity":"INFO", "time":"2020-04-22T16:04:50.691Z","correlation_id":"04f1366e-57a1-45b8-88c1-b00b23dc3616","class":"Projects::ImportExport::ExportService","current_user":"John Doe","project_full_path":"group1/test-export","file_path":"/path/to/archive","gc_stats":{"count":{"before":127,"after":127,"diff":0},"heap_allocated_pages":{"before":10369,"after":10369,"diff":0},"heap_sorted_length":{"before":10369,"after":10369,"diff":0},"heap_allocatable_pages":{"before":0,"after":0,"diff":0},"heap_available_slots":{"before":4226409,"after":4226409,"diff":0},"heap_live_slots":{"before":2542709,"after":2641420,"diff":98711},"heap_free_slots":{"before":1683700,"after":1584989,"diff":-98711},"heap_final_slots":{"before":0,"after":0,"diff":0},"heap_marked_slots":{"before":2542704,"after":2542704,"diff":0},"heap_eden_pages":{"before":10369,"after":10369,"diff":0},"heap_tomb_pages":{"before":0,"after":0,"diff":0},"total_allocated_pages":{"before":10369,"after":10369,"diff":0},"total_freed_pages":{"before":0,"after":0,"diff":0},"total_allocated_objects":{"before":24896308,"after":24995019,"diff":98711},"total_freed_objects":{"before":22353599,"after":22353599,"diff":0},"malloc_increase_bytes":{"before":140032,"after":6650240,"diff":6510208},"malloc_increase_bytes_limit":{"before":25804104,"after":25804104,"diff":0},"minor_gc_count":{"before":94,"after":94,"diff":0},"major_gc_count":{"before":33,"after":33,"diff":0},"remembered_wb_unprotected_objects":{"before":34284,"after":34284,"diff":0},"remembered_wb_unprotected_objects_limit":{"before":68568,"after":68568,"diff":0},"old_objects":{"before":2404725,"after":2404725,"diff":0},"old_objects_limit":{"before":4809450,"after":4809450,"diff":0},"oldmalloc_increase_bytes":{"before":140032,"after":6650240,"diff":6510208},"oldmalloc_increase_bytes_limit":{"before":68537556,"after":68537556,"diff":0}},"time_to_finish":0.12298400001600385,"number_of_sql_calls":70,"memory_usage":"0.0 MiB","label":"process_48616"} +``` + +## `geo.log` **(PREMIUM ONLY)** > Introduced in 9.5. @@ -677,7 +793,9 @@ For Omnibus installations, NGINX logs reside in: - `/var/log/gitlab/nginx/gitlab_pages_access.log` contains a log of requests made to Pages static sites. - `/var/log/gitlab/nginx/gitlab_pages_error.log` contains a log of NGINX errors for Pages static sites. - `/var/log/gitlab/nginx/gitlab_registry_access.log` contains a log of requests made to the Container Registry. -- `/var/log/gitlab/nginx/gitlab_registry_error.log` contains a log of NGINX errors for the Container Regsitry. +- `/var/log/gitlab/nginx/gitlab_registry_error.log` contains a log of NGINX errors for the Container Registry. +- `/var/log/gitlab/nginx/gitlab_mattermost_access.log` contains a log of requests made to Mattermost. +- `/var/log/gitlab/nginx/gitlab_mattermost_error.log` contains a log of NGINX errors for Mattermost. Below is the default GitLab NGINX access log format: @@ -685,7 +803,51 @@ Below is the default GitLab NGINX access log format: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ``` -[repocheck]: repository_checks.md -[Rack Attack]: ../security/rack_attack.md -[Rate Limit]: ../user/admin_area/settings/rate_limits_on_raw_endpoints.md -[Protected paths]: ../user/admin_area/settings/protected_paths.md +## Pages Logs + +For Omnibus installations, Pages logs reside in `/var/log/gitlab/gitlab-pages/current`. + +For example: + +```json +{ + "level": "info", + "msg": "GitLab Pages Daemon", + "revision": "52b2899", + "time": "2020-04-22T17:53:12Z", + "version": "1.17.0" +} +{ + "level": "info", + "msg": "URL: https://gitlab.com/gitlab-org/gitlab-pages", + "time": "2020-04-22T17:53:12Z" +} +{ + "gid": 998, + "in-place": false, + "level": "info", + "msg": "running the daemon as unprivileged user", + "time": "2020-04-22T17:53:12Z", + "uid": 998 +} +``` + +## Workhorse Logs + +For Omnibus installations, Workhorse logs reside in `/var/log/gitlab/gitlab-workhorse/current`. + +## PostgreSQL Logs + +For Omnibus installations, PostgreSQL logs reside in `/var/log/gitlab/postgresql/current`. + +## Prometheus Logs + +For Omnibus installations, Prometheus logs reside in `/var/log/gitlab/prometheus/current`. + +## Redis Logs + +For Omnibus installations, Redis logs reside in `/var/log/gitlab/redis/current`. + +## Mattermost Logs + +For Omnibus installations, Mattermost logs reside in `/var/log/gitlab/mattermost/mattermost.log`. diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index 884bb44cfda..091da46f316 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -15,7 +15,11 @@ All administrators at the time of creation of the project and group will be adde as maintainers of the group and project, and as an admin, you'll be able to add new members to the group in order to give them maintainer access to the project. -This project will be used for self monitoring your GitLab instance. +This project is used to self monitor your GitLab instance. Metrics are not yet +fully integrated, and the dashboard does not aggregate any data on Omnibus installations. GitLab plans +to provide integrated self-monitoring metrics in a future release. You can +currently use the project to configure your own [custom metrics](../../../user/project/integrations/prometheus.md#adding-custom-metrics) using +metrics exposed by the [GitLab exporter](../prometheus/gitlab_metrics.md#metrics-available). ## Creating the self monitoring project @@ -42,7 +46,7 @@ The project will be automatically configured to connect to the instance is present (should be the case if GitLab was installed via Omnibus and you haven't disabled it). -If that's not the case or if you have an external Prometheus instance or an HA setup, +If that's not the case or if you have an external Prometheus instance or a customized setup, you should [configure it manually](../../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus). @@ -70,7 +74,7 @@ project creation to fail with the following error (which appears in the log file when the first admin user is an [external user](../../../user/permissions.md#external-users-core-only): -```text +```plaintext Could not create instance administrators group. Errors: ["You don’t have permission to create groups."] ``` diff --git a/doc/administration/monitoring/performance/gitlab_configuration.md b/doc/administration/monitoring/performance/gitlab_configuration.md index e8a6c661464..14119a5d8f3 100644 --- a/doc/administration/monitoring/performance/gitlab_configuration.md +++ b/doc/administration/monitoring/performance/gitlab_configuration.md @@ -1,17 +1,9 @@ # GitLab Configuration -CAUTION: **InfluxDB is deprecated in favor of Prometheus:** -InfluxDB support is scheduled to be removed in GitLab 13.0. -You are advised to use [Prometheus](../prometheus/index.md) instead. - GitLab Performance Monitoring is disabled by default. To enable it and change any of its settings, navigate to **Admin Area > Settings > Metrics and profiling** (`/admin/application_settings/metrics_and_profiling`). -The minimum required settings you need to set are the InfluxDB host and port. -Make sure _Enable InfluxDB Metrics_ is checked and hit **Save** to save the -changes. -  Finally, a restart of all GitLab processes is required for the changes to take @@ -33,6 +25,4 @@ have been performed. Read more on: - [Introduction to GitLab Performance Monitoring](index.md) -- [InfluxDB Configuration](influxdb_configuration.md) -- [InfluxDB Schema](influxdb_schema.md) - [Grafana Install/Configuration](grafana_configuration.md) diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 3c26e399c7f..3438a564d2f 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -1,21 +1,12 @@ # Grafana Configuration -CAUTION: **InfluxDB is deprecated in favor of Prometheus:** -InfluxDB support is scheduled to be removed in GitLab 13.0. -You are advised to use [Prometheus](../prometheus/index.md) instead. - [Grafana](https://grafana.com/) is a tool that allows you to visualize time -series metrics through graphs and dashboards. It supports several backend -data stores, including InfluxDB. GitLab writes performance data to InfluxDB +series metrics through graphs and dashboards. GitLab writes performance data to Prometheus and Grafana will allow you to query to display useful graphs. -For the easiest installation and configuration, install Grafana on the same -server as InfluxDB. For larger installations, you may want to split out these -services. - ## Installation -[GitLab Omnibus can help you install Grafana (recommended)](https://docs.gitlab.com/omnibus/settings/grafana.html) +[Omnibus GitLab can help you install Grafana (recommended)](https://docs.gitlab.com/omnibus/settings/grafana.html) or Grafana supplies package repositories (Yum/Apt) for easy installation. See [Grafana installation documentation](https://grafana.com/docs/grafana/latest/installation/) for detailed steps. @@ -33,49 +24,8 @@ in the top bar.  -Fill in the configuration details for the InfluxDB data source. Save and -Test Connection to ensure the configuration is correct. - -- **Name**: `InfluxDB` -- **Default**: Checked -- **Type**: `InfluxDB 0.9.x` (Even if you're using InfluxDB 0.10.x) -- For the URL, use `https://localhost:8086`, or provide the remote URL if you've installed InfluxDB - on a separate server -- **Access**: `proxy` -- **Database**: `gitlab` -- **User**: `admin` (Or the username configured when setting up InfluxDB) -- **Password**: The password configured when you set up InfluxDB -  -## Apply retention policies and create continuous queries - -If you intend to import the GitLab provided Grafana dashboards, you will need to -set up the right retention policies and continuous queries. The easiest way of -doing this is by using the [InfluxDB Management](https://gitlab.com/gitlab-org/influxdb-management) -repository. - -To use this repository you must first clone it: - -```shell -git clone https://gitlab.com/gitlab-org/influxdb-management.git -cd influxdb-management -``` - -Next you must install the required dependencies: - -```shell -gem install bundler -bundle install -``` - -Now you must configure the repository by first copying `.env.example` to `.env` -and then editing the `.env` file to contain the correct InfluxDB settings. Once -configured you can simply run `bundle exec rake` and the InfluxDB database will -be configured for you. - -For more information see the [InfluxDB Management README](https://gitlab.com/gitlab-org/influxdb-management/blob/master/README.md). - ## Import Dashboards You can now import a set of default dashboards that will give you a good @@ -164,5 +114,3 @@ Read more on: - [Introduction to GitLab Performance Monitoring](index.md) - [GitLab Configuration](gitlab_configuration.md) -- [InfluxDB Installation/Configuration](influxdb_configuration.md) -- [InfluxDB Schema](influxdb_schema.md) diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md index 3305ea33f2e..02070287611 100644 --- a/doc/administration/monitoring/performance/index.md +++ b/doc/administration/monitoring/performance/index.md @@ -1,9 +1,5 @@ # GitLab Performance Monitoring -CAUTION: **InfluxDB is deprecated in favor of Prometheus:** -InfluxDB support is scheduled to be removed in GitLab 13.0. -You are advised to use [Prometheus](../prometheus/index.md) instead. - GitLab comes with its own application performance measuring system as of GitLab 8.4, simply called "GitLab Performance Monitoring". GitLab Performance Monitoring is available in both the Community and Enterprise editions. @@ -12,17 +8,11 @@ Apart from this introduction, you are advised to read through the following documents in order to understand and properly configure GitLab Performance Monitoring: - [GitLab Configuration](gitlab_configuration.md) -- [InfluxDB Install/Configuration](influxdb_configuration.md) -- [InfluxDB Schema](influxdb_schema.md) +- [Prometheus documentation](../prometheus/index.md) - [Grafana Install/Configuration](grafana_configuration.md) - [Performance bar](performance_bar.md) - [Request profiling](request_profiling.md) -NOTE: **Note:** -Omnibus GitLab 8.16 includes Prometheus as an additional tool to collect -metrics. It will eventually replace InfluxDB when their metrics collection is -on par. Read more in the [Prometheus documentation](../prometheus/index.md). - ## Introduction to GitLab Performance Monitoring GitLab Performance Monitoring makes it possible to measure a wide variety of statistics @@ -35,12 +25,6 @@ including (but not limited to): - System statistics such as the process' memory usage and open file descriptors. - Ruby garbage collection statistics. -Metrics data is written to [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) -over [UDP](https://docs.influxdata.com/influxdb/v0.9/write_protocols/udp/). -Stored data can be visualized using [Grafana](https://grafana.com) or any other application that -supports reading data from InfluxDB. Alternatively data can be queried using the -InfluxDB CLI. - ## Metric Types Two types of metrics are collected: diff --git a/doc/administration/monitoring/performance/influxdb_configuration.md b/doc/administration/monitoring/performance/influxdb_configuration.md index 8478272897b..d793e014a18 100644 --- a/doc/administration/monitoring/performance/influxdb_configuration.md +++ b/doc/administration/monitoring/performance/influxdb_configuration.md @@ -1,191 +1,5 @@ -# InfluxDB Configuration - -CAUTION: **InfluxDB is deprecated in favor of Prometheus:** -InfluxDB support is scheduled to be removed in GitLab 13.0. -You are advised to use [Prometheus](../prometheus/index.md) instead. - -The default settings provided by [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) -are not sufficient for a high traffic GitLab environment. The settings discussed in -this document are based on the settings GitLab uses for GitLab.com. Depending on -your own needs, you may need to further adjust them. - -If you are intending to run InfluxDB on the same server as GitLab, make sure -you have plenty of RAM since InfluxDB can use quite a bit depending on traffic. - -Unless you are going with a budget setup, it's advised to run it separately. - -## Requirements - -- InfluxDB 0.9.5 or newer -- A fairly modern version of Linux -- At least 4GB of RAM -- At least 10GB of storage for InfluxDB data - -Note that the RAM and storage requirements can differ greatly depending on the -amount of data received/stored. To limit the amount of stored data users can -look into -[InfluxDB Retention Policies](https://docs.influxdata.com/influxdb/v0.9/query_language/database_management/#retention-policy-management). - -## Installation - -Installing InfluxDB is out of the scope of this document. Please refer to the -[InfluxDB documentation](https://docs.influxdata.com/influxdb/v0.9/). - -## InfluxDB Server Settings - -Since InfluxDB has many settings that users may wish to customize themselves -(e.g. what port to run InfluxDB on), we'll only cover the essentials. - -The configuration file in question is usually located at -`/etc/influxdb/influxdb.conf`. Whenever you make a change in this file, -InfluxDB needs to be restarted. - -### Storage Engine - -InfluxDB comes with different storage engines and as of InfluxDB 0.9.5 a new -storage engine is available, called [TSM Tree](https://www.influxdata.com/blog/new-storage-engine-time-structured-merge-tree/). -All users **must** use the new `tsm1` storage engine as this -[will be the default engine](https://github.com/influxdata/influxdb/commit/15d723dc77651bac83e09e2b1c94be480966cb0d) in -upcoming InfluxDB releases. - -Make sure you have the following in your configuration file: - -```toml -[data] - dir = "/var/lib/influxdb/data" - engine = "tsm1" -``` - -### Admin Panel - -Production environments should have the InfluxDB admin panel **disabled**. This -feature can be disabled by adding the following to your InfluxDB configuration -file: - -```toml -[admin] - enabled = false -``` - -### HTTP - -HTTP is required when using the [InfluxDB CLI](https://docs.influxdata.com/influxdb/v0.9/tools/shell/) -or other tools such as Grafana, thus it should be enabled. When enabling -make sure to _also_ enable authentication: - -```toml -[http] - enabled = true - auth-enabled = true -``` - -_**Note:** Before you enable authentication, you might want to [create an -admin user](#create-a-new-admin-user)._ - -### UDP - -GitLab writes data to InfluxDB via UDP and thus this must be enabled. Enabling -UDP can be done using the following settings: - -```toml -[[udp]] - enabled = true - bind-address = ":8089" - database = "gitlab" - batch-size = 1000 - batch-pending = 5 - batch-timeout = "1s" - read-buffer = 209715200 -``` - -This does the following: - -1. Enable UDP and bind it to port 8089 for all addresses. -1. Store any data received in the `gitlab` database. -1. Define a batch of points to be 1000 points in size and allow a maximum of - 5 batches _or_ flush them automatically after 1 second. -1. Define a UDP read buffer size of 200 MB. - -One of the most important settings here is the UDP read buffer size as if this -value is set too low, packets will be dropped. You must also make sure the OS -buffer size is set to the same value, the default value is almost never enough. - -To set the OS buffer size to 200 MB, on Linux you can run the following command: - -```shell -sysctl -w net.core.rmem_max=209715200 -``` - -To make this permanent, add the following to `/etc/sysctl.conf` and restart the -server: - -```shell -net.core.rmem_max=209715200 -``` - -It is **very important** to make sure the buffer sizes are large enough to -handle all data sent to InfluxDB as otherwise you _will_ lose data. The above -buffer sizes are based on the traffic for GitLab.com. Depending on the amount of -traffic, users may be able to use a smaller buffer size, but we highly recommend -using _at least_ 100 MB. - -When enabling UDP, users should take care to not expose the port to the public, -as doing so will allow anybody to write data into your InfluxDB database (as -[InfluxDB's UDP protocol](https://docs.influxdata.com/influxdb/v0.9/write_protocols/udp/) -doesn't support authentication). We recommend either -whitelisting the allowed IP addresses/ranges, or setting up a VLAN and only -allowing traffic from members of said VLAN. - -## Create a new admin user - -If you want to [enable authentication](#http), you might want to [create an -admin user](https://docs.influxdata.com/influxdb/v0.9/administration/authentication_and_authorization/#create-a-new-admin-user): - -```shell -influx -execute "CREATE USER jeff WITH PASSWORD '1234' WITH ALL PRIVILEGES" -``` - -## Create the `gitlab` database - -Once you get InfluxDB up and running, you need to create a database for GitLab. -Make sure you have changed the [storage engine](#storage-engine) to `tsm1` -before creating a database. - -_**Note:** If you [created an admin user](#create-a-new-admin-user) and enabled -[HTTP authentication](#http), remember to append the username (`-username <username>`) -and password (`-password <password>`) you set earlier to the commands below._ - -Run the following command to create a database named `gitlab`: - -```shell -influx -execute 'CREATE DATABASE gitlab' -``` - -The name **must** be `gitlab`, do not use any other name. - -Next, make sure that the database was successfully created: - -```shell -influx -execute 'SHOW DATABASES' -``` - -The output should be similar to: - -```plaintext -name: databases ---------------- -name -_internal -gitlab -``` - -That's it! Now your GitLab instance should send data to InfluxDB. - +--- +redirect_to: 'prometheus.md' --- -Read more on: - -- [Introduction to GitLab Performance Monitoring](index.md) -- [GitLab Configuration](gitlab_configuration.md) -- [InfluxDB Schema](influxdb_schema.md) -- [Grafana Install/Configuration](grafana_configuration.md) +Support for InfluxDB was removed in GitLab 13.0. Use [Prometheus](prometheus.md) for performance monitoring. diff --git a/doc/administration/monitoring/performance/influxdb_schema.md b/doc/administration/monitoring/performance/influxdb_schema.md index adbccdaaeb8..d793e014a18 100644 --- a/doc/administration/monitoring/performance/influxdb_schema.md +++ b/doc/administration/monitoring/performance/influxdb_schema.md @@ -1,101 +1,5 @@ -# InfluxDB Schema - -CAUTION: **InfluxDB is deprecated in favor of Prometheus:** -InfluxDB support is scheduled to be removed in GitLab 13.0. -You are advised to use [Prometheus](../prometheus/index.md) instead. - -The following measurements are currently stored in InfluxDB: - -- `PROCESS_file_descriptors` -- `PROCESS_gc_statistics` -- `PROCESS_memory_usage` -- `PROCESS_method_calls` -- `PROCESS_object_counts` -- `PROCESS_transactions` -- `PROCESS_views` -- `events` - -Here, `PROCESS` is replaced with either `rails` or `sidekiq` depending on the -process type. In all series, any form of duration is stored in milliseconds. - -## PROCESS_file_descriptors - -This measurement contains the number of open file descriptors over time. The -value field `value` contains the number of descriptors. - -## PROCESS_gc_statistics - -This measurement contains Ruby garbage collection statistics such as the amount -of minor/major GC runs (relative to the last sampling interval), the time spent -in garbage collection cycles, and all fields/values returned by `GC.stat`. - -## PROCESS_memory_usage - -This measurement contains the process' memory usage (in bytes) over time. The -value field `value` contains the number of bytes. - -## PROCESS_method_calls - -This measurement contains the methods called during a transaction along with -their duration, and a name of the transaction action that invoked the method (if -available). The method call duration is stored in the value field `duration`, -while the method name is stored in the tag `method`. The tag `action` contains -the full name of the transaction action. Both the `method` and `action` fields -are in the following format: - -```plaintext -ClassName#method_name -``` - -For example, a method called by the `show` method in the `UsersController` class -would have `action` set to `UsersController#show`. - -## PROCESS_object_counts - -This measurement is used to store retained Ruby objects (per class) and the -amount of retained objects. The number of objects is stored in the `count` value -field while the class name is stored in the `type` tag. - -## PROCESS_transactions - -This measurement is used to store basic transaction details such as the time it -took to complete a transaction, how much time was spent in SQL queries, etc. The -following value fields are available: - -| Value | Description | -| ----- | ----------- | -| `duration` | The total duration of the transaction | -| `allocated_memory` | The amount of bytes allocated while the transaction was running. This value is only reliable when using single-threaded application servers | -| `method_duration` | The total time spent in method calls | -| `sql_duration` | The total time spent in SQL queries | -| `view_duration` | The total time spent in views | - -## PROCESS_views - -This measurement is used to store view rendering timings for a transaction. The -following value fields are available: - -| Value | Description | -| ----- | ----------- | -| `duration` | The rendering time of the view | -| `view` | The path of the view, relative to the application's root directory | - -The `action` tag contains the action name of the transaction that rendered the -view. - -## events - -This measurement is used to store generic events such as the number of Git -pushes, Emails sent, etc. Each point in this measurement has a single value -field called `count`. The value of this field is simply set to `1`. Each point -also has at least one tag: `event`. This tag's value is set to the event name. -Depending on the event type additional tags may be available as well. - +--- +redirect_to: 'prometheus.md' --- -Read more on: - -- [Introduction to GitLab Performance Monitoring](index.md) -- [GitLab Configuration](gitlab_configuration.md) -- [InfluxDB Configuration](influxdb_configuration.md) -- [Grafana Install/Configuration](grafana_configuration.md) +Support for InfluxDB was removed in GitLab 13.0. Use [Prometheus](prometheus.md) for performance monitoring. diff --git a/doc/administration/monitoring/prometheus/gitlab_exporter.md b/doc/administration/monitoring/prometheus/gitlab_exporter.md index 9c5c67e7f67..3effca4a2bf 100644 --- a/doc/administration/monitoring/prometheus/gitlab_exporter.md +++ b/doc/administration/monitoring/prometheus/gitlab_exporter.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab exporter >- Available since [Omnibus GitLab 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1132). diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 0833412ecb8..f725db9a039 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Prometheus metrics >**Note:** @@ -31,16 +37,17 @@ The following metrics are available: | Metric | Type | Since | Description | Labels | |:---------------------------------------------------------------|:----------|-----------------------:|:----------------------------------------------------------------------------------------------------|:----------------------------------------------------| -| `gitlab_banzai_cached_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output exists | controller, action | -| `gitlab_banzai_cacheless_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output does not exist | controller, action | -| `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | controller, action | +| `gitlab_banzai_cached_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output exists | `controller`, `action` | +| `gitlab_banzai_cacheless_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output does not exist | `controller`, `action` | +| `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | `controller`, `action` | | `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | | -| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller/action | controller, action, operation | -| `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | worker | -| `job_waiter_timeouts_total` | Counter | 12.9 | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete | worker | +| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller/action | `controller`, `action`, `operation` | +| `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | | +| `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` | +| `job_waiter_timeouts_total` | Counter | 12.9 | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete | `worker` | | `gitlab_database_transaction_seconds` | Histogram | 12.1 | Time spent in database transactions, in seconds | | -| `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | controller, action, module, method | -| `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | controller, action, bot | +| `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | `controller`, `action`, `module`, `method` | +| `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | `controller`, `action`, `bot` | | `gitlab_rails_queue_duration_seconds` | Histogram | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | | | `gitlab_sql_duration_seconds` | Histogram | 10.2 | SQL execution time, excluding SCHEMA operations and BEGIN / COMMIT | | | `gitlab_transaction_allocated_memory_bytes` | Histogram | 10.2 | Allocated memory for all transactions (gitlab_transaction_* metrics) | | @@ -48,27 +55,27 @@ The following metrics are available: | `gitlab_transaction_cache_<key>_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (per key) | | | `gitlab_transaction_cache_count_total` | Counter | 10.2 | Counter for total Rails cache calls (aggregate) | | | `gitlab_transaction_cache_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (aggregate) | | -| `gitlab_transaction_cache_read_hit_count_total` | Counter | 10.2 | Counter for cache hits for Rails cache calls | controller, action | -| `gitlab_transaction_cache_read_miss_count_total` | Counter | 10.2 | Counter for cache misses for Rails cache calls | controller, action | -| `gitlab_transaction_duration_seconds` | Histogram | 10.2 | Duration for all transactions (gitlab_transaction_* metrics) | controller, action | +| `gitlab_transaction_cache_read_hit_count_total` | Counter | 10.2 | Counter for cache hits for Rails cache calls | `controller`, `action` | +| `gitlab_transaction_cache_read_miss_count_total` | Counter | 10.2 | Counter for cache misses for Rails cache calls | `controller`, `action` | +| `gitlab_transaction_duration_seconds` | Histogram | 10.2 | Duration for all transactions (gitlab_transaction_* metrics) | `controller`, `action` | | `gitlab_transaction_event_build_found_total` | Counter | 9.4 | Counter for build found for API /jobs/request | | | `gitlab_transaction_event_build_invalid_total` | Counter | 9.4 | Counter for build invalid due to concurrency conflict for API /jobs/request | | | `gitlab_transaction_event_build_not_found_cached_total` | Counter | 9.4 | Counter for cached response of build not found for API /jobs/request | | | `gitlab_transaction_event_build_not_found_total` | Counter | 9.4 | Counter for build not found for API /jobs/request | | | `gitlab_transaction_event_change_default_branch_total` | Counter | 9.4 | Counter when default branch is changed for any repository | | | `gitlab_transaction_event_create_repository_total` | Counter | 9.4 | Counter when any repository is created | | -| `gitlab_transaction_event_etag_caching_cache_hit_total` | Counter | 9.4 | Counter for etag cache hit. | endpoint | -| `gitlab_transaction_event_etag_caching_header_missing_total` | Counter | 9.4 | Counter for etag cache miss - header missing | endpoint | -| `gitlab_transaction_event_etag_caching_key_not_found_total` | Counter | 9.4 | Counter for etag cache miss - key not found | endpoint | -| `gitlab_transaction_event_etag_caching_middleware_used_total` | Counter | 9.4 | Counter for etag middleware accessed | endpoint | -| `gitlab_transaction_event_etag_caching_resource_changed_total` | Counter | 9.4 | Counter for etag cache miss - resource changed | endpoint | +| `gitlab_transaction_event_etag_caching_cache_hit_total` | Counter | 9.4 | Counter for etag cache hit. | `endpoint` | +| `gitlab_transaction_event_etag_caching_header_missing_total` | Counter | 9.4 | Counter for etag cache miss - header missing | `endpoint` | +| `gitlab_transaction_event_etag_caching_key_not_found_total` | Counter | 9.4 | Counter for etag cache miss - key not found | `endpoint` | +| `gitlab_transaction_event_etag_caching_middleware_used_total` | Counter | 9.4 | Counter for etag middleware accessed | `endpoint` | +| `gitlab_transaction_event_etag_caching_resource_changed_total` | Counter | 9.4 | Counter for etag cache miss - resource changed | `endpoint` | | `gitlab_transaction_event_fork_repository_total` | Counter | 9.4 | Counter for repository forks (RepositoryForkWorker). Only incremented when source repository exists | | | `gitlab_transaction_event_import_repository_total` | Counter | 9.4 | Counter for repository imports (RepositoryImportWorker) | | | `gitlab_transaction_event_push_branch_total` | Counter | 9.4 | Counter for all branch pushes | | -| `gitlab_transaction_event_push_commit_total` | Counter | 9.4 | Counter for commits | branch | +| `gitlab_transaction_event_push_commit_total` | Counter | 9.4 | Counter for commits | `branch` | | `gitlab_transaction_event_push_tag_total` | Counter | 9.4 | Counter for tag pushes | | | `gitlab_transaction_event_rails_exception_total` | Counter | 9.4 | Counter for number of rails exceptions | | -| `gitlab_transaction_event_receive_email_total` | Counter | 9.4 | Counter for received emails | handler | +| `gitlab_transaction_event_receive_email_total` | Counter | 9.4 | Counter for received emails | `handler` | | `gitlab_transaction_event_remote_mirrors_failed_total` | Counter | 10.8 | Counter for failed remote mirrors | | | `gitlab_transaction_event_remote_mirrors_finished_total` | Counter | 10.8 | Counter for finished remote mirrors | | | `gitlab_transaction_event_remote_mirrors_running_total` | Counter | 10.8 | Counter for running remote mirrors | | @@ -76,15 +83,15 @@ The following metrics are available: | `gitlab_transaction_event_remove_repository_total` | Counter | 9.4 | Counter when a repository is removed | | | `gitlab_transaction_event_remove_tag_total` | Counter | 9.4 | Counter when a tag is remove for any repository | | | `gitlab_transaction_event_sidekiq_exception_total` | Counter | 9.4 | Counter of Sidekiq exceptions | | -| `gitlab_transaction_event_stuck_import_jobs_total` | Counter | 9.4 | Count of stuck import jobs | projects_without_jid_count, projects_with_jid_count | -| `gitlab_transaction_event_update_build_total` | Counter | 9.4 | Counter for update build for API /jobs/request/:id | | +| `gitlab_transaction_event_stuck_import_jobs_total` | Counter | 9.4 | Count of stuck import jobs | `projects_without_jid_count`, `projects_with_jid_count` | +| `gitlab_transaction_event_update_build_total` | Counter | 9.4 | Counter for update build for API `/jobs/request/:id` | | | `gitlab_transaction_new_redis_connections_total` | Counter | 9.4 | Counter for new Redis connections | | | `gitlab_transaction_queue_duration_total` | Counter | 9.4 | Duration jobs were enqueued before processing | | -| `gitlab_transaction_rails_queue_duration_total` | Counter | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | controller, action | -| `gitlab_transaction_view_duration_total` | Counter | 9.4 | Duration for views | controller, action, view | -| `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | controller, action, view | -| `http_requests_total` | Counter | 9.4 | Rack request count | method | -| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | method, status | +| `gitlab_transaction_rails_queue_duration_total` | Counter | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | `controller`, `action` | +| `gitlab_transaction_view_duration_total` | Counter | 9.4 | Duration for views | `controller`, `action`, `view` | +| `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | `controller`, `action`, `view` | +| `http_requests_total` | Counter | 9.4 | Rack request count | `method` | +| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | `method`, `status` | | `pipelines_created_total` | Counter | 9.4 | Counter of pipelines created | | | `rack_uncaught_errors_total` | Counter | 9.4 | Rack connections handling uncaught errors count | | | `user_session_logins_total` | Counter | 9.4 | Counter of how many users have logged in | | @@ -112,49 +119,52 @@ configuration option in `gitlab.yml`. These metrics are served from the | Metric | Type | Since | Description | Labels | |:---------------------------------------------- |:------- |:----- |:----------- |:------ | -| `sidekiq_jobs_cpu_seconds` | Histogram | 12.4 | Seconds of cpu time to run Sidekiq job | queue, boundary, external_dependencies, feature_category, job_status, urgency | -| `sidekiq_jobs_completion_seconds` | Histogram | 12.2 | Seconds to complete Sidekiq job | queue, boundary, external_dependencies, feature_category, job_status, urgency | -| `sidekiq_jobs_db_seconds` | Histogram | 12.9 | Seconds of DB time to run Sidekiq job | queue, boundary, external_dependencies, feature_category, job_status, urgency | -| `sidekiq_jobs_gitaly_seconds` | Histogram | 12.9 | Seconds of Gitaly time to run Sidekiq job | queue, boundary, external_dependencies, feature_category, job_status, urgency | -| `sidekiq_jobs_queue_duration_seconds` | Histogram | 12.5 | Duration in seconds that a Sidekiq job was queued before being executed | queue, boundary, external_dependencies, feature_category, urgency | -| `sidekiq_jobs_failed_total` | Counter | 12.2 | Sidekiq jobs failed | queue, boundary, external_dependencies, feature_category, urgency | -| `sidekiq_jobs_retried_total` | Counter | 12.2 | Sidekiq jobs retried | queue, boundary, external_dependencies, feature_category, urgency | -| `sidekiq_running_jobs` | Gauge | 12.2 | Number of Sidekiq jobs running | queue, boundary, external_dependencies, feature_category, urgency | +| `sidekiq_jobs_cpu_seconds` | Histogram | 12.4 | Seconds of cpu time to run Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_jobs_completion_seconds` | Histogram | 12.2 | Seconds to complete Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_jobs_db_seconds` | Histogram | 12.9 | Seconds of DB time to run Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_jobs_gitaly_seconds` | Histogram | 12.9 | Seconds of Gitaly time to run Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_jobs_queue_duration_seconds` | Histogram | 12.5 | Duration in seconds that a Sidekiq job was queued before being executed | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | +| `sidekiq_jobs_failed_total` | Counter | 12.2 | Sidekiq jobs failed | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | +| `sidekiq_jobs_retried_total` | Counter | 12.2 | Sidekiq jobs retried | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | +| `sidekiq_running_jobs` | Gauge | 12.2 | Number of Sidekiq jobs running | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | | `sidekiq_concurrency` | Gauge | 12.5 | Maximum number of Sidekiq jobs | | -| `geo_db_replication_lag_seconds` | Gauge | 10.2 | Database replication lag (seconds) | url | -| `geo_repositories` | Gauge | 10.2 | Total number of repositories available on primary | url | -| `geo_repositories_synced` | Gauge | 10.2 | Number of repositories synced on secondary | url | -| `geo_repositories_failed` | Gauge | 10.2 | Number of repositories failed to sync on secondary | url | -| `geo_lfs_objects` | Gauge | 10.2 | Total number of LFS objects available on primary | url | -| `geo_lfs_objects_synced` | Gauge | 10.2 | Number of LFS objects synced on secondary | url | -| `geo_lfs_objects_failed` | Gauge | 10.2 | Number of LFS objects failed to sync on secondary | url | -| `geo_attachments` | Gauge | 10.2 | Total number of file attachments available on primary | url | -| `geo_attachments_synced` | Gauge | 10.2 | Number of attachments synced on secondary | url | -| `geo_attachments_failed` | Gauge | 10.2 | Number of attachments failed to sync on secondary | url | -| `geo_last_event_id` | Gauge | 10.2 | Database ID of the latest event log entry on the primary | url | -| `geo_last_event_timestamp` | Gauge | 10.2 | UNIX timestamp of the latest event log entry on the primary | url | -| `geo_cursor_last_event_id` | Gauge | 10.2 | Last database ID of the event log processed by the secondary | url | -| `geo_cursor_last_event_timestamp` | Gauge | 10.2 | Last UNIX timestamp of the event log processed by the secondary | url | -| `geo_status_failed_total` | Counter | 10.2 | Number of times retrieving the status from the Geo Node failed | url | -| `geo_last_successful_status_check_timestamp` | Gauge | 10.2 | Last timestamp when the status was successfully updated | url | -| `geo_lfs_objects_synced_missing_on_primary` | Gauge | 10.7 | Number of LFS objects marked as synced due to the file missing on the primary | url | -| `geo_job_artifacts_synced_missing_on_primary` | Gauge | 10.7 | Number of job artifacts marked as synced due to the file missing on the primary | url | -| `geo_attachments_synced_missing_on_primary` | Gauge | 10.7 | Number of attachments marked as synced due to the file missing on the primary | url | -| `geo_repositories_checksummed_count` | Gauge | 10.7 | Number of repositories checksummed on primary | url | -| `geo_repositories_checksum_failed_count` | Gauge | 10.7 | Number of repositories failed to calculate the checksum on primary | url | -| `geo_wikis_checksummed_count` | Gauge | 10.7 | Number of wikis checksummed on primary | url | -| `geo_wikis_checksum_failed_count` | Gauge | 10.7 | Number of wikis failed to calculate the checksum on primary | url | -| `geo_repositories_verified_count` | Gauge | 10.7 | Number of repositories verified on secondary | url | -| `geo_repositories_verification_failed_count` | Gauge | 10.7 | Number of repositories failed to verify on secondary | url | -| `geo_repositories_checksum_mismatch_count` | Gauge | 10.7 | Number of repositories that checksum mismatch on secondary | url | -| `geo_wikis_verified_count` | Gauge | 10.7 | Number of wikis verified on secondary | url | -| `geo_wikis_verification_failed_count` | Gauge | 10.7 | Number of wikis failed to verify on secondary | url | -| `geo_wikis_checksum_mismatch_count` | Gauge | 10.7 | Number of wikis that checksum mismatch on secondary | url | -| `geo_repositories_checked_count` | Gauge | 11.1 | Number of repositories that have been checked via `git fsck` | url | -| `geo_repositories_checked_failed_count` | Gauge | 11.1 | Number of repositories that have a failure from `git fsck` | url | -| `geo_repositories_retrying_verification_count` | Gauge | 11.2 | Number of repositories verification failures that Geo is actively trying to correct on secondary | url | -| `geo_wikis_retrying_verification_count` | Gauge | 11.2 | Number of wikis verification failures that Geo is actively trying to correct on secondary | url | +| `geo_db_replication_lag_seconds` | Gauge | 10.2 | Database replication lag (seconds) | `url` | +| `geo_repositories` | Gauge | 10.2 | Total number of repositories available on primary | `url` | +| `geo_repositories_synced` | Gauge | 10.2 | Number of repositories synced on secondary | `url` | +| `geo_repositories_failed` | Gauge | 10.2 | Number of repositories failed to sync on secondary | `url` | +| `geo_lfs_objects` | Gauge | 10.2 | Total number of LFS objects available on primary | `url` | +| `geo_lfs_objects_synced` | Gauge | 10.2 | Number of LFS objects synced on secondary | `url` | +| `geo_lfs_objects_failed` | Gauge | 10.2 | Number of LFS objects failed to sync on secondary | `url` | +| `geo_attachments` | Gauge | 10.2 | Total number of file attachments available on primary | `url` | +| `geo_attachments_synced` | Gauge | 10.2 | Number of attachments synced on secondary | `url` | +| `geo_attachments_failed` | Gauge | 10.2 | Number of attachments failed to sync on secondary | `url` | +| `geo_last_event_id` | Gauge | 10.2 | Database ID of the latest event log entry on the primary | `url` | +| `geo_last_event_timestamp` | Gauge | 10.2 | UNIX timestamp of the latest event log entry on the primary | `url` | +| `geo_cursor_last_event_id` | Gauge | 10.2 | Last database ID of the event log processed by the secondary | `url` | +| `geo_cursor_last_event_timestamp` | Gauge | 10.2 | Last UNIX timestamp of the event log processed by the secondary | `url` | +| `geo_status_failed_total` | Counter | 10.2 | Number of times retrieving the status from the Geo Node failed | `url` | +| `geo_last_successful_status_check_timestamp` | Gauge | 10.2 | Last timestamp when the status was successfully updated | `url` | +| `geo_lfs_objects_synced_missing_on_primary` | Gauge | 10.7 | Number of LFS objects marked as synced due to the file missing on the primary | `url` | +| `geo_job_artifacts_synced_missing_on_primary` | Gauge | 10.7 | Number of job artifacts marked as synced due to the file missing on the primary | `url` | +| `geo_attachments_synced_missing_on_primary` | Gauge | 10.7 | Number of attachments marked as synced due to the file missing on the primary | `url` | +| `geo_repositories_checksummed_count` | Gauge | 10.7 | Number of repositories checksummed on primary | `url` | +| `geo_repositories_checksum_failed_count` | Gauge | 10.7 | Number of repositories failed to calculate the checksum on primary | `url` | +| `geo_wikis_checksummed_count` | Gauge | 10.7 | Number of wikis checksummed on primary | `url` | +| `geo_wikis_checksum_failed_count` | Gauge | 10.7 | Number of wikis failed to calculate the checksum on primary | `url` | +| `geo_repositories_verified_count` | Gauge | 10.7 | Number of repositories verified on secondary | `url` | +| `geo_repositories_verification_failed_count` | Gauge | 10.7 | Number of repositories failed to verify on secondary | `url` | +| `geo_repositories_checksum_mismatch_count` | Gauge | 10.7 | Number of repositories that checksum mismatch on secondary | `url` | +| `geo_wikis_verified_count` | Gauge | 10.7 | Number of wikis verified on secondary | `url` | +| `geo_wikis_verification_failed_count` | Gauge | 10.7 | Number of wikis failed to verify on secondary | `url` | +| `geo_wikis_checksum_mismatch_count` | Gauge | 10.7 | Number of wikis that checksum mismatch on secondary | `url` | +| `geo_repositories_checked_count` | Gauge | 11.1 | Number of repositories that have been checked via `git fsck` | `url` | +| `geo_repositories_checked_failed_count` | Gauge | 11.1 | Number of repositories that have a failure from `git fsck` | `url` | +| `geo_repositories_retrying_verification_count` | Gauge | 11.2 | Number of repositories verification failures that Geo is actively trying to correct on secondary | `url` | +| `geo_wikis_retrying_verification_count` | Gauge | 11.2 | Number of wikis verification failures that Geo is actively trying to correct on secondary | `url` | | `global_search_bulk_cron_queue_size` | Gauge | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | | +| `package_files_count` | Gauge | 13.0 | Number of package files on primary | `url` | +| `package_files_checksummed_count` | Gauge | 13.0 | Number of package files checksummed on primary | `url` | +| `package_files_checksum_failed_count` | Gauge | 13.0 | Number of package files failed to calculate the checksum on primary ## Database load balancing metrics **(PREMIUM ONLY)** @@ -168,17 +178,18 @@ The following metrics are available: Some basic Ruby runtime metrics are available: -| Metric | Type | Since | Description | -|:------------------------------------ |:--------- |:----- |:----------- | -| `ruby_gc_duration_seconds` | Counter | 11.1 | Time spent by Ruby in GC | -| `ruby_gc_stat_...` | Gauge | 11.1 | Various metrics from [GC.stat](https://ruby-doc.org/core-2.6.5/GC.html#method-c-stat) | -| `ruby_file_descriptors` | Gauge | 11.1 | File descriptors per process | -| `ruby_memory_bytes` | Gauge | 11.1 | Memory usage by process | -| `ruby_sampler_duration_seconds` | Counter | 11.1 | Time spent collecting stats | -| `ruby_process_cpu_seconds_total` | Gauge | 12.0 | Total amount of CPU time per process | -| `ruby_process_max_fds` | Gauge | 12.0 | Maximum number of open file descriptors per process | -| `ruby_process_resident_memory_bytes` | Gauge | 12.0 | Memory usage by process | -| `ruby_process_start_time_seconds` | Gauge | 12.0 | UNIX timestamp of process start time | +| Metric | Type | Since | Description | +|:---------------------------------------- |:--------- |:----- |:----------- | +| `ruby_gc_duration_seconds` | Counter | 11.1 | Time spent by Ruby in GC | +| `ruby_gc_stat_...` | Gauge | 11.1 | Various metrics from [GC.stat](https://ruby-doc.org/core-2.6.5/GC.html#method-c-stat) | +| `ruby_file_descriptors` | Gauge | 11.1 | File descriptors per process | +| `ruby_sampler_duration_seconds` | Counter | 11.1 | Time spent collecting stats | +| `ruby_process_cpu_seconds_total` | Gauge | 12.0 | Total amount of CPU time per process | +| `ruby_process_max_fds` | Gauge | 12.0 | Maximum number of open file descriptors per process | +| `ruby_process_resident_memory_bytes` | Gauge | 12.0 | Memory usage by process (RSS/Resident Set Size) | +| `ruby_process_unique_memory_bytes` | Gauge | 13.0 | Memory usage by process (USS/Unique Set Size) | +| `ruby_process_proportional_memory_bytes` | Gauge | 13.0 | Memory usage by process (PSS/Proportional Set Size) | +| `ruby_process_start_time_seconds` | Gauge | 12.0 | UNIX timestamp of process start time | ## Unicorn Metrics diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 03d86012cc7..cb93aca6e4e 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring GitLab with Prometheus > **Notes:** @@ -49,7 +55,7 @@ To disable Prometheus and all of its exporters, as well as any added in the futu ### Changing the port and address Prometheus listens on NOTE: **Note:** -The following change was added in [GitLab Omnibus 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1261). Although possible, +The following change was added in [Omnibus GitLab 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1261). Although possible, it's not recommended to change the port Prometheus listens on, as this might affect or conflict with other services running on the GitLab server. Proceed at your own risk. @@ -80,7 +86,7 @@ To change the address/port that Prometheus listens on: ### Adding custom scrape configs -You can configure additional scrape targets for the GitLab Omnibus-bundled +You can configure additional scrape targets for the Omnibus GitLab-bundled Prometheus by editing `prometheus['scrape_configs']` in `/etc/gitlab/gitlab.rb` using the [Prometheus scrape target configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cscrape_config%3E) syntax. @@ -108,7 +114,7 @@ prometheus['scrape_configs'] = [ NOTE: **Note:** Prometheus and most exporters don't support authentication. We don't recommend exposing them outside the local network. -A few configuration changes are required to allow GitLab to be monitored by an external Prometheus server. External servers are recommended for highly available deployments of GitLab with multiple nodes. +A few configuration changes are required to allow GitLab to be monitored by an external Prometheus server. External servers are recommended for [GitLab deployments with multiple nodes](../../reference_architectures/index.md). To use an external Prometheus server: diff --git a/doc/administration/monitoring/prometheus/node_exporter.md b/doc/administration/monitoring/prometheus/node_exporter.md index d75b04f1ccd..357303ee4e1 100644 --- a/doc/administration/monitoring/prometheus/node_exporter.md +++ b/doc/administration/monitoring/prometheus/node_exporter.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Node exporter >**Note:** diff --git a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md index ba8e464efcb..92ba2d9bb52 100644 --- a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md +++ b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # PgBouncer exporter >**Note:** diff --git a/doc/administration/monitoring/prometheus/postgres_exporter.md b/doc/administration/monitoring/prometheus/postgres_exporter.md index 853e3837184..77ca502b21d 100644 --- a/doc/administration/monitoring/prometheus/postgres_exporter.md +++ b/doc/administration/monitoring/prometheus/postgres_exporter.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # PostgreSQL Server Exporter >**Note:** diff --git a/doc/administration/monitoring/prometheus/redis_exporter.md b/doc/administration/monitoring/prometheus/redis_exporter.md index 76f4add0c1b..bef87400f5a 100644 --- a/doc/administration/monitoring/prometheus/redis_exporter.md +++ b/doc/administration/monitoring/prometheus/redis_exporter.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Redis exporter >**Note:** diff --git a/doc/administration/monitoring/prometheus/registry_exporter.md b/doc/administration/monitoring/prometheus/registry_exporter.md index 2e2440389ed..3d28b26b685 100644 --- a/doc/administration/monitoring/prometheus/registry_exporter.md +++ b/doc/administration/monitoring/prometheus/registry_exporter.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Registry exporter > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2884) in GitLab 11.9. diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 1b0c7f8a907..c4f9b672923 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -5,7 +5,7 @@ type: reference # Object Storage GitLab supports using an object storage service for holding numerous types of data. -In a high availability setup, it's recommended over [NFS](high_availability/nfs.md) and +It's recommended over NFS and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. @@ -37,8 +37,8 @@ For configuring GitLab to use Object Storage refer to the following guides: ### Other alternatives to filesystem storage -If you're working to [scale out](scaling/index.md) your GitLab implementation, -or add [fault tolerance and redundancy](high_availability/README.md) you may be +If you're working to [scale out](reference_architectures/index.md) your GitLab implementation, +or add fault tolerance and redundancy, you may be looking at removing dependencies on block or network filesystems. See the following guides and [note that Pages requires disk storage](#gitlab-pages-requires-nfs): @@ -77,7 +77,7 @@ with the Fog library that GitLab uses. Symptoms include: ### GitLab Pages requires NFS -If you're working to add more GitLab servers for [scaling or fault tolerance](scaling/index.md) +If you're working to add more GitLab servers for [scaling or fault tolerance](reference_architectures/index.md) and one of your requirements is [GitLab Pages](../user/project/pages/index.md) this currently requires NFS. There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) to remove this dependency. In the future, GitLab Pages may use @@ -98,9 +98,9 @@ object storage back end, like when Git clients request large files via LFS or wh downloading CI artifacts and logs. When the files are stored on local block storage or NFS, GitLab has to act as a proxy. -This is not the default behaviour with object storage. +This is not the default behavior with object storage. -The `proxy_download` setting controls this behaviour: the default is generally `false`. +The `proxy_download` setting controls this behavior: the default is generally `false`. Verify this in the documentation for each use case. Set it to `true` so that GitLab proxies the files. diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 1c92a429982..8f54b82c325 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -1,13 +1,13 @@ -# Running multiple Sidekiq processes **(CORE ONLY)** - -NOTE: **Note:** -The information in this page applies only to Omnibus GitLab. +# Run multiple Sidekiq processes **(CORE ONLY)** GitLab allows you to start multiple Sidekiq processes. These processes can be used to consume a dedicated set of queues. This can be used to ensure certain queues always have dedicated workers, no matter the number of jobs that need to be processed. +NOTE: **Note:** +The information in this page applies only to Omnibus GitLab. + ## Available Sidekiq queues For a list of the existing Sidekiq queues, check the following files: @@ -18,28 +18,27 @@ For a list of the existing Sidekiq queues, check the following files: Each entry in the above files represents a queue on which Sidekiq processes can be started. -## Starting multiple processes +## Start multiple processes -To start multiple Sidekiq processes, you must enable `sidekiq-cluster`: +> - [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4006) in GitLab 12.10, starting multiple processes with Sidekiq cluster. +> - [Sidekiq cluster moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab [Core](https://about.gitlab.com/pricing/#self-managed) in GitLab 12.10. +> - [Sidekiq cluster became default](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4140) in GitLab 13.0. -1. Edit `/etc/gitlab/gitlab.rb` and add: +To start multiple processes: - ```ruby - sidekiq_cluster['enable'] = true - ``` - -1. You will then need to specify how many additional processes to create via `sidekiq-cluster` - and which queue they should handle via the `sidekiq_cluster['queue_groups']` - array setting. Each item in the array equates to one additional Sidekiq +1. Using the `sidekiq['queue_groups']` array setting, specify how many processes to + create using `sidekiq-cluster` and which queue they should handle. + Each item in the array equates to one additional Sidekiq process, and values in each item determine the queues it works on. - For example, the following setting adds additional Sidekiq processes to two - queues, one to `elastic_indexer` and one to `mailers`: + For example, the following setting creates three Sidekiq processes, one to run on + `elastic_indexer`, one to run on `mailers`, and one process running all on queues: ```ruby - sidekiq_cluster['queue_groups'] = [ + sidekiq['queue_groups'] = [ "elastic_indexer", - "mailers" + "mailers", + "*" ] ``` @@ -47,9 +46,10 @@ To start multiple Sidekiq processes, you must enable `sidekiq-cluster`: queue names to its item delimited by commas. For example: ```ruby - sidekiq_cluster['queue_groups'] = [ + sidekiq['queue_groups'] = [ "elastic_indexer, elastic_commit_indexer", - "mailers" + "mailers", + "*" ] ``` @@ -58,7 +58,7 @@ To start multiple Sidekiq processes, you must enable `sidekiq-cluster`: processes, each handling all queues: ```ruby - sidekiq_cluster['queue_groups'] = [ + sidekiq['queue_groups'] = [ "*", "*" ] @@ -67,27 +67,35 @@ To start multiple Sidekiq processes, you must enable `sidekiq-cluster`: `*` cannot be combined with concrete queue names - `*, mailers` will just handle the `mailers` queue. + When `sidekiq-cluster` is only running on a single node, make sure that at least + one process is running on all queues using `*`. This means a process will + automatically pick up jobs in queues created in the future. + + If `sidekiq-cluster` is running on more than one node, you can also use + [`--negate`](#negate-settings) and list all the queues that are already being + processed. + 1. Save the file and reconfigure GitLab for the changes to take effect: ```shell sudo gitlab-ctl reconfigure ``` -Once the extra Sidekiq processes are added, you can visit the -**Admin Area > Monitoring > Background Jobs** (`/admin/background_jobs`) in GitLab. +After the extra Sidekiq processes are added, navigate to +**{admin}** **Admin Area > Monitoring > Background Jobs** (`/admin/background_jobs`) in GitLab.  -## Negating settings +## Negate settings To have the additional Sidekiq processes work on every queue **except** the ones you list: -1. After you follow the steps for [starting extra processes](#starting-multiple-processes), +1. After you follow the steps for [starting extra processes](#start-multiple-processes), edit `/etc/gitlab/gitlab.rb` and add: ```ruby - sidekiq_cluster['negate'] = true + sidekiq['negate'] = true ``` 1. Save the file and reconfigure GitLab for the changes to take effect: @@ -177,9 +185,9 @@ entire queue group selects all queues. In `/etc/gitlab/gitlab.rb`: ```ruby -sidekiq_cluster['enable'] = true -sidekiq_cluster['experimental_queue_selector'] = true -sidekiq_cluster['queue_groups'] = [ +sidekiq['enable'] = true +sidekiq['experimental_queue_selector'] = true +sidekiq['queue_groups'] = [ # Run all non-CPU-bound queues that are high urgency 'resource_boundary!=cpu&urgency=high', # Run all continuous integration and pages queues that are not high urgency @@ -189,35 +197,31 @@ sidekiq_cluster['queue_groups'] = [ ] ``` -### Using Sidekiq cluster by default (experimental) - -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4006) in GitLab 12.10. +### Disable Sidekiq cluster CAUTION: **Warning:** -This feature is experimental. +Sidekiq cluster is [scheduled](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/240) +to be the only way to start Sidekiq in GitLab 14.0. -We're moving [Sidekiq cluster to -core](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) and -plan to make it the default way of starting Sidekiq. - -Set the following to start Sidekiq (cluster) -process for handling for all queues (`/etc/gitlab/gitlab.rb`): +By default, the Sidekiq service will run `sidekiq-cluster`. To disable this behavior, +add the following to the Sidekiq configuration: ```ruby sidekiq['enable'] = true -sidekiq['cluster'] = true +sidekiq['cluster'] = false ``` -All of the aforementioned configuration options for `sidekiq_cluster` -are also available. By default, they will be configured as follows: +All of the aforementioned configuration options for `sidekiq` +are available. By default, they will be configured as follows: ```ruby sidekiq['experimental_queue_selector'] = false sidekiq['interval'] = nil -sidekiq['max_concurrency'] = nil +sidekiq['max_concurrency'] = 50 sidekiq['min_concurrency'] = nil sidekiq['negate'] = false sidekiq['queue_groups'] = ['*'] +sidekiq['shutdown_timeout'] = 25 ``` `sidekiq_cluster` must be disabled if you decide to configure the @@ -231,7 +235,7 @@ setting `sidekiq['cluster'] = true`. When using this feature, the service called `sidekiq` will now be running `sidekiq-cluster`. -The [concurrency](#managing-concurrency) and other options configured +The [concurrency](#manage-concurrency) and other options configured for Sidekiq will be respected. By default, logs for `sidekiq-cluster` go to `/var/log/gitlab/sidekiq` @@ -246,9 +250,9 @@ use all of its resources to perform those operations. To set up a separate 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - sidekiq_cluster['enable'] = true - sidekiq_cluster['negate'] = true - sidekiq_cluster['queue_groups'] = [ + sidekiq['enable'] = true + sidekiq['negate'] = true + sidekiq['queue_groups'] = [ "github_import_advance_stage", "github_importer:github_import_import_diff_note", "github_importer:github_import_import_issue", @@ -274,12 +278,12 @@ use all of its resources to perform those operations. To set up a separate ## Number of threads -Each process defined under `sidekiq_cluster` starts with a +Each process defined under `sidekiq` starts with a number of threads that equals the number of queues, plus one spare thread. For example, a process that handles the `process_commit` and `post_receive` queues will use three threads in total. -## Managing concurrency +## Manage concurrency When setting the maximum concurrency, keep in mind this normally should not exceed the number of CPU cores available. The values in the examples @@ -290,29 +294,15 @@ latency and potentially cause client timeouts. See the [Sidekiq documentation about Redis](https://github.com/mperham/sidekiq/wiki/Using-Redis) for more details. -### When running a single Sidekiq process (default) - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['concurrency'] = 25 - ``` +### When running Sidekiq cluster (default) -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -This will set the concurrency (number of threads) for the Sidekiq process. - -### When running Sidekiq cluster +Running Sidekiq cluster is the default in GitLab 13.0 and later. 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - sidekiq_cluster['min_concurrency'] = 15 - sidekiq_cluster['max_concurrency'] = 25 + sidekiq['min_concurrency'] = 15 + sidekiq['max_concurrency'] = 25 ``` 1. Save the file and reconfigure GitLab for the changes to take effect: @@ -337,21 +327,44 @@ regardless of the number of queues. When `min_concurrency` is greater than `max_concurrency`, it is treated as being equal to `max_concurrency`. -## Modifying the check interval +### When running a single Sidekiq process + +Running a single Sidekiq process is the default in GitLab 12.10 and earlier. + +CAUTION: **Warning:** +Running Sidekiq directly is scheduled to be removed in GitLab +[14.0](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/240). + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + sidekiq['cluster'] = false + sidekiq['concurrency'] = 25 + ``` + +1. Save the file and reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +This will set the concurrency (number of threads) for the Sidekiq process. + +## Modify the check interval To modify the check interval for the additional Sidekiq processes: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - sidekiq_cluster['interval'] = 5 + sidekiq['interval'] = 5 ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. This tells the additional processes how often to check for enqueued jobs. -## Troubleshooting using the CLI +## Troubleshoot using the CLI CAUTION: **Warning:** It's recommended to use `/etc/gitlab/gitlab.rb` to configure the Sidekiq processes. @@ -399,7 +412,7 @@ you'd use the following: /opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive gitlab_shell ``` -### Monitoring the `sidekiq-cluster` command +### Monitor the `sidekiq-cluster` command The `sidekiq-cluster` command will not terminate once it has started the desired amount of Sidekiq processes. Instead, the process will continue running and @@ -412,7 +425,7 @@ processes will terminate themselves after a few seconds. This ensures you don't end up with zombie Sidekiq processes. All of this makes monitoring the processes fairly easy. Simply hook up -`sidekiq-cluster` to your supervisor of choice (e.g. runit) and you're good to +`sidekiq-cluster` to your supervisor of choice (for example, runit) and you're good to go. If a child process died the `sidekiq-cluster` command will signal all remaining diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 2d1e1c5bda8..6759c3f265d 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -68,11 +68,17 @@ sudo service sshd reload ``` Confirm that SSH is working by removing your user's SSH key in the UI, adding a -new one, and attempting to pull a repo. +new one, and attempting to pull a repository. NOTE: **Note:** For Omnibus Docker, `AuthorizedKeysCommand` is setup by default in GitLab 11.11 and later. +NOTE: **Note:** For Installations from source, the command would be located at +`/home/git/gitlab-shell/bin/gitlab-shell-authorized-keys-check` if [the install from source](../../install/installation.md#install-gitlab-shell) instructions were followed. +You might want to consider creating a wrapper script somewhere else since this command needs to be +owned by `root` and not be writable by group or others. You could also consider changing the ownership of this command +as required, but that might require temporary ownership changes during `gitlab-shell` upgrades. + CAUTION: **Caution:** Do not disable writes until SSH is confirmed to be working perfectly, because the file will quickly become out-of-date. @@ -87,7 +93,7 @@ installation.  Again, confirm that SSH is working by removing your user's SSH key in the UI, -adding a new one, and attempting to pull a repo. +adding a new one, and attempting to pull a repository. Then you can backup and delete your `authorized_keys` file for best performance. diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index c27832e67ef..45b8e5ad448 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -12,6 +12,7 @@ Keep your GitLab instance up and running smoothly. - [Sidekiq MemoryKiller](sidekiq_memory_killer.md): Configure Sidekiq MemoryKiller to restart Sidekiq. - [Multiple Sidekiq processes](extra_sidekiq_processes.md): Configure multiple Sidekiq processes to ensure certain queues always have dedicated workers, no matter the number of jobs that need to be processed. **(CORE ONLY)** +- [Puma](puma.md): Understand Puma and puma-worker-killer. - [Unicorn](unicorn.md): Understand Unicorn and unicorn-worker-killer. - Speed up SSH operations by [Authorizing SSH users via a fast, indexed lookup to the GitLab database](fast_ssh_key_lookup.md), and/or diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index 6f252a7d76e..af559cf00e9 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -3,7 +3,9 @@ ## Puma As of GitLab 12.9, [Puma](https://github.com/puma/puma) has replaced [Unicorn](https://yhbt.net/unicorn/). -as the default web server. +as the default web server. Starting with 13.0, both all-in-one package based +installations as well as Helm chart based installations will run Puma instead of +Unicorn unless explicitly specified not to. ## Why switch to Puma? @@ -14,6 +16,12 @@ Most Rails applications requests normally include a proportion of I/O wait time. During I/O wait time MRI Ruby will release the GVL (Global VM Lock) to other threads. Multi-threaded Puma can therefore still serve more requests than a single process. +## Configuring Puma to replace Unicorn + +If you are currently running Unicorn and would like to switch to Puma, server configuration +will _not_ carry over automatically. For details on matching Unicorn configuration settings with +the Puma equivalent, where applicable, see [Converting Unicorn settings to Puma](https://docs.gitlab.com/omnibus/settings/puma.html#converting-unicorn-settings-to-puma). + ## Performance caveat when using Puma with Rugged For deployments where NFS is used to store Git repository, we allow GitLab to use diff --git a/doc/administration/operations/sidekiq_memory_killer.md b/doc/administration/operations/sidekiq_memory_killer.md index 6438dbb9dab..fdccfacc8a9 100644 --- a/doc/administration/operations/sidekiq_memory_killer.md +++ b/doc/administration/operations/sidekiq_memory_killer.md @@ -2,13 +2,13 @@ The GitLab Rails application code suffers from memory leaks. For web requests this problem is made manageable using -[`unicorn-worker-killer`](https://github.com/kzk/unicorn-worker-killer) which -restarts Unicorn worker processes in between requests when needed. The Sidekiq +[`puma-worker-killer`](https://github.com/schneems/puma_worker_killer) which +restarts Puma worker processes if it exceeds a memory limit. The Sidekiq MemoryKiller applies the same approach to the Sidekiq processes used by GitLab to process background jobs. -Unlike unicorn-worker-killer, which is enabled by default for all GitLab -installations since GitLab 6.4, the Sidekiq MemoryKiller is enabled by default +Unlike puma-worker-killer, which is enabled by default for all GitLab +installations since GitLab 13.0, the Sidekiq MemoryKiller is enabled by default _only_ for Omnibus packages. The reason for this is that the MemoryKiller relies on runit to restart Sidekiq after a memory-induced shutdown and GitLab installations from source do not all use runit or an equivalent. diff --git a/doc/administration/operations/ssh_certificates.md b/doc/administration/operations/ssh_certificates.md index eaf0e4ab284..b652f282b7b 100644 --- a/doc/administration/operations/ssh_certificates.md +++ b/doc/administration/operations/ssh_certificates.md @@ -50,7 +50,7 @@ the GitLab server itself, but your setup may vary. If the CA is only used for GitLab consider putting this in the `Match User git` section (described below). -The SSH certificates being issued by that CA **MUST** have a "key id" +The SSH certificates being issued by that CA **MUST** have a "key ID" corresponding to that user's username on GitLab, e.g. (some output omitted for brevity): @@ -77,7 +77,7 @@ own `AuthorizedPrincipalsCommand` to do that mapping instead of using our provided default. The important part is that the `AuthorizedPrincipalsCommand` must be -able to map from the "key id" to a GitLab username in some way, the +able to map from the "key ID" to a GitLab username in some way, the default command we ship assumes there's a 1=1 mapping between the two, since the whole point of this is to allow us to extract a GitLab username from the key itself, instead of relying on something like the @@ -122,7 +122,7 @@ into multiple lines of `authorized_keys` output, as described in the Normally when using the `AuthorizedKeysCommand` with OpenSSH the principal is some "group" that's allowed to log into that server. However with GitLab it's only used to appease OpenSSH's -requirement for it, we effectively only care about the "key id" being +requirement for it, we effectively only care about the "key ID" being correct. Once that's extracted GitLab will enforce its own ACLs for that user (e.g. what projects the user can access). diff --git a/doc/administration/operations/unicorn.md b/doc/administration/operations/unicorn.md index bb817e71f5a..50481482f4c 100644 --- a/doc/administration/operations/unicorn.md +++ b/doc/administration/operations/unicorn.md @@ -1,5 +1,9 @@ # Understanding Unicorn and unicorn-worker-killer +NOTE: **Note:** +Starting with GitLab 13.0, Puma is the default web server used in GitLab +all-in-one package based installations as well as GitLab Helm chart deployments. + ## Unicorn GitLab uses [Unicorn](https://yhbt.net/unicorn/), a pre-forking Ruby web diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index aaf1ca29084..503e5fb4a2a 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -18,16 +18,22 @@ You can read more about the Docker Registry at **Omnibus GitLab installations** -If you are using the Omnibus GitLab built in [Let's Encrypt integration](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration), as of GitLab 12.5, the Container Registry will be automatically enabled on port 5050 of the default domain. +If you installed GitLab by using the Omnibus installation package, the Container Registry +may or may not be available by default. -If you would like to use a separate domain, all you have to do is configure the domain name under which the Container -Registry will listen to. Read -[#container-registry-domain-configuration](#container-registry-domain-configuration) -and pick one of the two options that fits your case. +The Container Registry is automatically enabled and available on your GitLab domain, port 5050 if: + +- You're using the built-in [Let's Encrypt integration](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration), and +- You're using GitLab 12.5 or later. + +Otherwise, the Container Registry is not enabled. To enable it: + +- You can configure it for your [GitLab domain](#configure-container-registry-under-an-existing-gitlab-domain), or +- You can configure it for [a different domain](#configure-container-registry-under-its-own-domain). NOTE: **Note:** -The container registry works under HTTPS by default. Using HTTP is possible -but not recommended and out of the scope of this document. +The Container Registry works under HTTPS by default. You can use HTTP +but it's not recommended and is out of the scope of this document. Read the [insecure Registry documentation](https://docs.docker.com/registry/insecure/) if you want to implement this. @@ -427,7 +433,7 @@ NOTE: **Note:** By default, users accessing a registry configured with a remote backend are redirected to the default backend for the storage driver. For example, registries can be configured using the `s3` storage driver, which redirects requests to a remote S3 bucket to alleviate load on the GitLab server. -However, this behaviour is undesirable for registries used by internal hosts that usually can't access public servers. To disable redirects, set the `disable` flag to true as follows. This makes all traffic to always go through the Registry service. This results in improved security (less surface attack as the storage backend is not publicly accessible), but worse performance (all traffic is redirected via the service). +However, this behavior is undesirable for registries used by internal hosts that usually can't access public servers. To disable redirects, set the `disable` flag to true as follows. This makes all traffic to always go through the Registry service. This results in improved security (less surface attack as the storage backend is not publicly accessible), but worse performance (all traffic is redirected via the service). **Omnibus GitLab installations** @@ -454,7 +460,7 @@ However, this behaviour is undesirable for registries used by internal hosts tha 1. Add the `redirect` flag to your registry configuration YML file: - ```yml + ```yaml storage: s3: accesskey: 'AKIAKIAKI' @@ -520,7 +526,7 @@ on how to achieve that. NOTE: **Note:** In using an external container registry, some features associated with the -container registry may be unavailable or have [inherant risks](./../../user/packages/container_registry/index.md#use-with-external-container-registries) +container registry may be unavailable or have [inherent risks](./../../user/packages/container_registry/index.md#use-with-external-container-registries) **Omnibus GitLab** @@ -650,13 +656,13 @@ notifications: NOTE: **Note:** The garbage collection tools are only available when you've installed GitLab -via an Omnibus package or the cloud native chart. +via an Omnibus package or the [cloud native chart](https://docs.gitlab.com/charts/charts/registry/#garbage-collection). DANGER: **Danger:** By running the built-in garbage collection command, it will cause downtime to -the Container Registry. Running this command on an instance in an HA environment -while one of your other instances is still writing to the Registry storage, -will remove referenced manifests. To avoid that, make sure Registry is set to +the Container Registry. If you run this command on an instance in an environment +where one of your other instances is still writing to the Registry storage, +referenced manifests will be removed. To avoid that, make sure Registry is set to [read-only mode](#performing-garbage-collection-without-downtime) before proceeding. Container Registry can use considerable amounts of disk space. To clear up @@ -710,7 +716,7 @@ built-in command: If you did not change the default location of the configuration file, run: -```sh +```shell sudo gitlab-ctl registry-garbage-collect ``` @@ -719,7 +725,7 @@ layers you have stored. If you changed the location of the Container Registry `config.yml`: -```sh +```shell sudo gitlab-ctl registry-garbage-collect /path/to/config.yml ``` @@ -743,7 +749,7 @@ referenced by the registry tag. The `registry-garbage-collect` command supports `-m` switch to allow you to remove all unreferenced manifests and layers that are not directly accessible via `tag`: -```sh +```shell sudo gitlab-ctl registry-garbage-collect -m ``` @@ -781,7 +787,7 @@ To enable the read-only mode: 1. Save and reconfigure GitLab: - ```sh + ```shell sudo gitlab-ctl reconfigure ``` @@ -789,7 +795,7 @@ To enable the read-only mode: 1. Next, trigger one of the garbage collect commands: - ```sh + ```shell # Recycling unused tags sudo /opt/gitlab/embedded/bin/registry garbage-collect /var/opt/gitlab/registry/config.yml @@ -816,7 +822,7 @@ To enable the read-only mode: 1. Save and reconfigure GitLab: - ```sh + ```shell sudo gitlab-ctl reconfigure ``` diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 84133205bd3..21d13be47bd 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -80,7 +80,7 @@ added `gitlab.io` [in 2016](https://gitlab.com/gitlab-com/infrastructure/issues/ ### DNS configuration GitLab Pages expect to run on their own virtual host. In your DNS server/provider -you need to add a [wildcard DNS A record][wiki-wildcard-dns] pointing to the +you need to add a [wildcard DNS A record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) pointing to the host that GitLab runs. For example, an entry would look like this: ```plaintext @@ -95,8 +95,6 @@ IPv6 address. If you don't have IPv6, you can omit the AAAA record. NOTE: **Note:** You should not use the GitLab domain to serve user pages. For more information see the [security section](#security). -[wiki-wildcard-dns]: https://en.wikipedia.org/wiki/Wildcard_DNS_record - ## Configuration Depending on your needs, you can set up GitLab Pages in 4 different ways. @@ -354,7 +352,7 @@ This usually results in this error: For installation from source this can be fixed by installing the custom Certificate Authority (CA) in the system certificate store. -For Omnibus, normally this would be fixed by [installing a custom CA in GitLab Omnibus](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) +For Omnibus, normally this would be fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) but a [bug](https://gitlab.com/gitlab-org/gitlab/issues/25411) is currently preventing that method from working. Use the following workaround: @@ -365,14 +363,14 @@ that method from working. Use the following workaround: echo -n | openssl s_client -connect gitlab-domain-example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee --append /opt/gitlab/embedded/ssl/certs/cacert.pem ``` -1. [Restart](../restart_gitlab.md) the GitLab Pages Daemon. For GitLab Omnibus instances: +1. [Restart](../restart_gitlab.md) the GitLab Pages Daemon. For Omnibus GitLab instances: ```shell sudo gitlab-ctl restart gitlab-pages ``` CAUTION: **Caution:** -Some GitLab Omnibus upgrades will revert this workaround and you'll need to apply it again. +Some Omnibus GitLab upgrades will revert this workaround and you'll need to apply it again. ## Activate verbose logging for daemon @@ -457,9 +455,36 @@ You can run the GitLab Pages daemon on a separate server in order to decrease th To configure GitLab Pages on a separate server: +DANGER: **Danger:** +The following procedure includes steps to back up and edit the +`gitlab-secrets.json` file. This file contains secrets that control +database encryption. Proceed with caution. + +1. On the **GitLab server**, to enable Pages, add the following to `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['enable'] = true + ``` + +1. Optionally, to enable [access control](#access-control), add the following to `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['access_control'] = true + ``` + +1. [Reconfigure the **GitLab server**](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the + changes to take effect. The `gitlab-secrets.json` file is now updated with the + new configuration. + +1. Create a backup of the secrets file on the **GitLab server**: + + ```shell + cp /etc/gitlab/gitlab-secrets.json /etc/gitlab/gitlab-secrets.json.bak + ``` + 1. Set up a new server. This will become the **Pages server**. -1. Create an NFS share on the new server and configure this share to +1. Create an [NFS share](../high_availability/nfs_host_client_setup.md) on the new server and configure this share to allow access from your main **GitLab server**. For this example, we use the default GitLab Pages folder `/var/opt/gitlab/gitlab-rails/shared/pages` as the shared folder on the new server and we will mount it to `/mnt/pages` @@ -474,7 +499,7 @@ To configure GitLab Pages on a separate server: postgresql['enable'] = false redis['enable'] = false prometheus['enable'] = false - unicorn['enable'] = false + puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false gitaly['enable'] = false @@ -483,6 +508,15 @@ To configure GitLab Pages on a separate server: gitlab_rails['auto_migrate'] = false ``` +1. Create a backup of the secrets file on the **Pages server**: + + ```shell + cp /etc/gitlab/gitlab-secrets.json /etc/gitlab/gitlab-secrets.json.bak + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the **GitLab server** + to the **Pages server**. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. On the **GitLab server**, make the following changes to `/etc/gitlab/gitlab.rb`: @@ -502,61 +536,6 @@ configuring a load balancer to work at the IP level, and so on. If you wish to set up GitLab Pages on multiple servers, perform the above procedure for each Pages server. -### Access control when running GitLab Pages on a separate server - -If you are [running GitLab Pages on a separate server](#running-gitlab-pages-on-a-separate-server), -then you must use the following procedure to configure [access control](#access-control): - -1. On the **GitLab server**, add the following to `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['enable'] = true - gitlab_pages['access_control'] = true - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the - changes to take effect. The `gitlab-secrets.json` file is now updated with the - new configuration. - - DANGER: **Danger:** - The `gitlab-secrets.json` file contains secrets that control database encryption. - Do not edit or replace this file on the **GitLab server** or you might - experience permanent data loss. Make a backup copy of this file before proceeding, - as explained in the following steps. - -1. Create a backup of the secrets file on the **GitLab server**: - - ```shell - cp /etc/gitlab/gitlab-secrets.json /etc/gitlab/gitlab-secrets.json.bak - ``` - -1. Create a backup of the secrets file on the **Pages server**: - - ```shell - cp /etc/gitlab/gitlab-secrets.json /etc/gitlab/gitlab-secrets.json.bak - ``` - -1. Disable Pages on the **GitLab server** by setting the following in - `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['enable'] = false - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the **GitLab server** - to the **Pages server**. - -1. On your **Pages server**, add the following to `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['gitlab_server'] = "https://<your-gitlab-server-URL>" - gitlab_pages['access_control'] = true - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - ## Backup GitLab Pages are part of the [regular backup](../../raketasks/backup_restore.md), so there is no separate backup to configure. diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index 94d2c5420aa..1bb3b86b419 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -61,7 +61,7 @@ Before proceeding with the Pages configuration, make sure that: ### DNS configuration GitLab Pages expect to run on their own virtual host. In your DNS server/provider -you need to add a [wildcard DNS A record][wiki-wildcard-dns] pointing to the +you need to add a [wildcard DNS A record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) pointing to the host that GitLab runs. For example, an entry would look like this: ```plaintext @@ -75,8 +75,6 @@ and `192.0.2.1` is the IP address of your GitLab instance. You should not use the GitLab domain to serve user pages. For more information see the [security section](#security). -[wiki-wildcard-dns]: https://en.wikipedia.org/wiki/Wildcard_DNS_record - ## Configuration Depending on your needs, you can set up GitLab Pages in 4 different ways. diff --git a/doc/administration/pseudonymizer.md b/doc/administration/pseudonymizer.md index 36bb446da78..22543a5c743 100644 --- a/doc/administration/pseudonymizer.md +++ b/doc/administration/pseudonymizer.md @@ -1,6 +1,6 @@ # Pseudonymizer **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5532) in [GitLab Ultimate][ee] 11.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5532) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.1. As GitLab's database hosts sensitive information, using it unfiltered for analytics implies high security requirements. To help alleviate this constraint, the Pseudonymizer @@ -101,5 +101,3 @@ This will produce some CSV files that might be very large, so make sure the After the pseudonymizer has run, the output CSV files should be uploaded to the configured object storage and deleted from the local disk. - -[ee]: https://about.gitlab.com/pricing/ diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index c0edb43cc01..da5caf3966f 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -1,6 +1,8 @@ -# Integrity Check Rake Task +# Integrity check Rake task **(CORE ONLY)** -## Repository Integrity +GitLab provides Rake tasks to check the integrity of various components. + +## Repository integrity Even though Git is very resilient and tries to prevent data integrity issues, there are times when things go wrong. The following Rake tasks intend to @@ -43,7 +45,7 @@ sudo gitlab-rake gitlab:git:fsck sudo -u git -H bundle exec rake gitlab:git:fsck RAILS_ENV=production ``` -## Uploaded Files Integrity +## Uploaded files integrity Various types of files can be uploaded to a GitLab installation by users. These integrity checks can detect missing files. Additionally, for locally @@ -127,11 +129,9 @@ Checking integrity of Uploads Done! ``` -## LDAP Check +## LDAP check The LDAP check Rake task will test the bind DN and password credentials (if configured) and will list a sample of LDAP users. This task is also executed as part of the `gitlab:check` task, but can run independently. See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details. - -[git-fsck]: https://git-scm.com/docs/git-fsck diff --git a/doc/administration/raketasks/geo.md b/doc/administration/raketasks/geo.md index 707ed1dbee0..71e4f922348 100644 --- a/doc/administration/raketasks/geo.md +++ b/doc/administration/raketasks/geo.md @@ -1,9 +1,11 @@ # Geo Rake Tasks **(PREMIUM ONLY)** +The following Rake tasks are for [Geo installations](../geo/replication/index.md). + ## Git housekeeping There are few tasks you can run to schedule a Git housekeeping to start at the -next repository sync in a **Secondary node**: +next repository sync in a **secondary** node: ### Incremental Repack diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md index 4c6521f38ec..83a3d2c8884 100644 --- a/doc/administration/raketasks/github_import.md +++ b/doc/administration/raketasks/github_import.md @@ -1,4 +1,4 @@ -# GitHub import +# GitHub import **(CORE ONLY)** > [Introduced]( https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10308) in GitLab 9.1. @@ -9,7 +9,7 @@ which will become the owner of the project. You can resume an import with the same command. Bear in mind that the syntax is very specific. Remove any spaces within the argument block and -before/after the brackets. Also, Some shells (e.g., zsh) can interpret the open/close brackets +before/after the brackets. Also, some shells (for example, `zsh`) can interpret the open/close brackets (`[]`) separately. You may need to either escape the brackets or use double quotes. ## Importing multiple projects diff --git a/doc/administration/raketasks/ldap.md b/doc/administration/raketasks/ldap.md index 361ab28f6d4..2871a9235a3 100644 --- a/doc/administration/raketasks/ldap.md +++ b/doc/administration/raketasks/ldap.md @@ -1,4 +1,6 @@ -# LDAP Rake Tasks +# LDAP Rake tasks **(CORE ONLY)** + +The following are LDAP-related Rake tasks. ## Check @@ -26,7 +28,7 @@ limit by passing a number to the check task: rake gitlab:ldap:check[50] ``` -## Run a Group Sync +## Run a group sync **(STARTER ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/14735) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.2. diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index c79a1aa6ba1..eee68c0da0a 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -1,8 +1,11 @@ -# Maintenance Rake Tasks +# Maintenance Rake tasks **(CORE ONLY)** -## Gather information about GitLab and the system it runs on +GitLab provides Rake tasks for general maintenance. -This command gathers information about your GitLab installation and the System it runs on. These may be useful when asking for help or reporting issues. +## Gather GitLab and system information + +This command gathers information about your GitLab installation and the system it runs on. +These may be useful when asking for help or reporting issues. **Omnibus Installation** @@ -50,20 +53,23 @@ Git: /usr/bin/git ## Check GitLab configuration -Runs the following Rake tasks: +The `gitlab:check` Rake task runs the following Rake tasks: - `gitlab:gitlab_shell:check` - `gitlab:gitaly:check` - `gitlab:sidekiq:check` - `gitlab:app:check` -It will check that each component was set up according to the installation guide and suggest fixes for issues found. -This command must be run from your app server and will not work correctly on component servers like [Gitaly](../gitaly/index.md#running-gitaly-on-its-own-server). +It will check that each component was set up according to the installation guide and suggest fixes +for issues found. This command must be run from your application server and will not work correctly on +component servers like [Gitaly](../gitaly/index.md#running-gitaly-on-its-own-server). + +You may also have a look at our troubleshooting guides for: -You may also have a look at our Troubleshooting Guides: +- [GitLab](../index.md#troubleshooting) +- [Omnibus GitLab](https://docs.gitlab.com/omnibus/README.html#troubleshooting) -- [Troubleshooting Guide (GitLab)](../index.md#troubleshooting) -- [Troubleshooting Guide (Omnibus GitLab)](https://docs.gitlab.com/omnibus/README.html#troubleshooting) +To run `gitlab:check`, run: **Omnibus Installation** @@ -77,7 +83,8 @@ sudo gitlab-rake gitlab:check bundle exec rake gitlab:check RAILS_ENV=production ``` -NOTE: Use `SANITIZE=true` for `gitlab:check` if you want to omit project names from the output. +NOTE: **Note:** +Use `SANITIZE=true` for `gitlab:check` if you want to omit project names from the output. Example output: @@ -126,7 +133,7 @@ Checking GitLab ... Finished ## Rebuild authorized_keys file -In some case it is necessary to rebuild the `authorized_keys` file. +In some case it is necessary to rebuild the `authorized_keys` file. To do this, run: **Omnibus Installation** @@ -141,6 +148,8 @@ cd /home/git/gitlab sudo -u git -H bundle exec rake gitlab:shell:setup RAILS_ENV=production ``` +Example output: + ```plaintext This will rebuild an authorized_keys file. You will lose any data stored in authorized_keys file. @@ -149,8 +158,8 @@ Do you want to continue (yes/no)? yes ## Clear Redis cache -If for some reason the dashboard shows wrong information you might want to -clear Redis' cache. +If for some reason the dashboard displays the wrong information, you might want to +clear Redis' cache. To do this, run: **Omnibus Installation** @@ -170,7 +179,7 @@ sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production Sometimes during version upgrades you might end up with some wrong CSS or missing some icons. In that case, try to precompile the assets again. -Note that this only applies to source installations and does NOT apply to +This only applies to source installations and does NOT apply to Omnibus packages. **Source Installation** @@ -187,25 +196,6 @@ production machine after installing the package, there should be no reason to re `rake gitlab:assets:compile` on the production machine. If you suspect that assets have been corrupted, you should reinstall the omnibus package. -## Tracking Deployments - -GitLab provides a Rake task that lets you track deployments in GitLab -Performance Monitoring. This Rake task simply stores the current GitLab version -in the GitLab Performance Monitoring database. - -**Omnibus Installation** - -```shell -sudo gitlab-rake gitlab:track_deployment -``` - -**Source Installation** - -```shell -cd /home/git/gitlab -sudo -u git -H bundle exec rake gitlab:track_deployment RAILS_ENV=production -``` - ## Check TCP connectivity to a remote site Sometimes you need to know if your GitLab installation can connect to a TCP diff --git a/doc/administration/raketasks/praefect.md b/doc/administration/raketasks/praefect.md index 4f9cf0e10ba..c3dadb6bc30 100644 --- a/doc/administration/raketasks/praefect.md +++ b/doc/administration/raketasks/praefect.md @@ -1,4 +1,4 @@ -# Praefect Rake Tasks +# Praefect Rake Tasks **(CORE ONLY)** > [Introduced]( https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28369) in GitLab 12.10. diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index 6d874d596e1..2ab8b13e29e 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -3,11 +3,13 @@ > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/3050) in GitLab 8.9. > - From GitLab 11.3, import/export can use object storage automatically. -See also: +GitLab provides Rake tasks relating to project import and export. For more information, see: - [Project import/export documentation](../../user/project/settings/import_export.md). - [Project import/export API](../../api/project_import_export.md). +## Import/export tasks + The GitLab import/export version can be checked by using the following command: ```shell @@ -28,8 +30,6 @@ sudo gitlab-rake gitlab:import_export:data bundle exec rake gitlab:import_export:data RAILS_ENV=production ``` -## Important notes - Note the following: - Importing is only possible if the version of the import and export GitLab instances are diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md index 1e487a317c9..30f50c24138 100644 --- a/doc/administration/raketasks/storage.md +++ b/doc/administration/raketasks/storage.md @@ -1,214 +1,162 @@ -# Repository Storage Rake Tasks +# Repository storage Rake tasks **(CORE ONLY)** -This is a collection of Rake tasks you can use to help you list and migrate -existing projects and attachments associated with it from Legacy storage to -the new Hashed storage type. +This is a collection of Rake tasks to help you list and migrate +existing projects and their attachments to the new +[hashed storage](../repository_storage_types.md) that GitLab +uses to organize the Git data. -You can read more about the storage types [here][storage-types]. +## List projects and attachments -## Migrate existing projects to Hashed storage +The following Rake tasks will list the projects and attachments that are +available on legacy and hashed storage. -Before migrating your existing projects, you should -[enable hashed storage][storage-migration] for the new projects as well. +### On legacy storage -This task will schedule all your existing projects and attachments associated with it to be migrated to the -**Hashed** storage type: +To have a summary and then a list of projects and their attachments using legacy storage: -**Omnibus Installation** +- **Omnibus installation** -```shell -sudo gitlab-rake gitlab:storage:migrate_to_hashed -``` - -**Source Installation** - -```shell -sudo -u git -H bundle exec rake gitlab:storage:migrate_to_hashed RAILS_ENV=production -``` - -They both also accept a range as environment variable: - -```shell -# to migrate any non migrated project from ID 20 to 50. -export ID_FROM=20 -export ID_TO=50 -``` - -You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. -There is a specific Queue you can watch to see how long it will take to finish: -`hashed_storage:hashed_storage_project_migrate` - -After it reaches zero, you can confirm every project has been migrated by running the commands bellow. -If you find it necessary, you can run this migration script again to schedule missing projects. - -Any error or warning will be logged in Sidekiq's log file. - -NOTE: **Note:** -If Geo is enabled, each project that is successfully migrated generates an event to replicate the changes on any **secondary** nodes. - -You only need the `gitlab:storage:migrate_to_hashed` Rake task to migrate your repositories, but we have additional -commands below that helps you inspect projects and attachments in both legacy and hashed storage. - -## Rollback from Hashed storage to Legacy storage - -If you need to rollback the storage migration for any reason, you can follow the steps described here. - -NOTE: **Note:** Hashed Storage will be required in future version of GitLab. - -To prevent new projects from being created in the Hashed storage, -you need to undo the [enable hashed storage][storage-migration] changes. - -This task will schedule all your existing projects and associated attachments to be rolled back to the -Legacy storage type. - -For Omnibus installations, run the following: - -```shell -sudo gitlab-rake gitlab:storage:rollback_to_legacy -``` - -For source installations, run the following: - -```shell -sudo -u git -H bundle exec rake gitlab:storage:rollback_to_legacy RAILS_ENV=production -``` - -Both commands accept a range as environment variable: - -```shell -# to rollback any migrated project from ID 20 to 50. -export ID_FROM=20 -export ID_TO=50 -``` - -You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. -On the **Queues** tab, you can watch the `hashed_storage:hashed_storage_project_rollback` queue to see how long the process will take to finish. - -After it reaches zero, you can confirm every project has been rolled back by running the commands bellow. -If some projects weren't rolled back, you can run this rollback script again to schedule further rollbacks. - -Any error or warning will be logged in Sidekiq's log file. - -## List projects on Legacy storage + ```shell + # Projects + sudo gitlab-rake gitlab:storage:legacy_projects + sudo gitlab-rake gitlab:storage:list_legacy_projects -To have a simple summary of projects using **Legacy** storage: + # Attachments + sudo gitlab-rake gitlab:storage:legacy_attachments + sudo gitlab-rake gitlab:storage:list_legacy_attachments + ``` -**Omnibus Installation** +- **Source installation** -```shell -sudo gitlab-rake gitlab:storage:legacy_projects -``` - -**Source Installation** - -```shell -sudo -u git -H bundle exec rake gitlab:storage:legacy_projects RAILS_ENV=production -``` + ```shell + # Projects + sudo -u git -H bundle exec rake gitlab:storage:legacy_projects RAILS_ENV=production + sudo -u git -H bundle exec rake gitlab:storage:list_legacy_projects RAILS_ENV=production -To list projects using **Legacy** storage: + # Attachments + sudo -u git -H bundle exec rake gitlab:storage:legacy_attachments RAILS_ENV=production + sudo -u git -H bundle exec rake gitlab:storage:list_legacy_attachments RAILS_ENV=production + ``` -**Omnibus Installation** - -```shell -sudo gitlab-rake gitlab:storage:list_legacy_projects -``` - -**Source Installation** - -```shell -sudo -u git -H bundle exec rake gitlab:storage:list_legacy_projects RAILS_ENV=production +### On hashed storage -``` +To have a summary and then a list of projects and their attachments using hashed storage: -## List projects on Hashed storage +- **Omnibus installation** -To have a simple summary of projects using **Hashed** storage: + ```shell + # Projects + sudo gitlab-rake gitlab:storage:hashed_projects + sudo gitlab-rake gitlab:storage:list_hashed_projects -**Omnibus Installation** + # Attachments + sudo gitlab-rake gitlab:storage:hashed_attachments + sudo gitlab-rake gitlab:storage:list_hashed_attachments + ``` -```shell -sudo gitlab-rake gitlab:storage:hashed_projects -``` +- **Source installation** -**Source Installation** + ```shell + # Projects + sudo -u git -H bundle exec rake gitlab:storage:hashed_projects RAILS_ENV=production + sudo -u git -H bundle exec rake gitlab:storage:list_hashed_projects RAILS_ENV=production -```shell -sudo -u git -H bundle exec rake gitlab:storage:hashed_projects RAILS_ENV=production -``` + # Attachments + sudo -u git -H bundle exec rake gitlab:storage:hashed_attachments RAILS_ENV=production + sudo -u git -H bundle exec rake gitlab:storage:list_hashed_attachments RAILS_ENV=production + ``` -To list projects using **Hashed** storage: +## Migrate to hashed storage -**Omnibus Installation** +NOTE: **Note:** +In GitLab 13.0, [hashed storage](../repository_storage_types.md#hashed-storage) +is enabled by default and the legacy storage is deprecated. +Support for legacy storage will be removed in GitLab 14.0. If you're on GitLab +13.0 and later, switching new projects to legacy storage is not possible. +The option to choose between hashed and legacy storage in the admin area has +been disabled. -```shell -sudo gitlab-rake gitlab:storage:list_hashed_projects -``` +This task will schedule all your existing projects and attachments associated +with it to be migrated to the **Hashed** storage type: -**Source Installation** +- **Omnibus installation** -```shell -sudo -u git -H bundle exec rake gitlab:storage:list_hashed_projects RAILS_ENV=production -``` + ```shell + sudo gitlab-rake gitlab:storage:migrate_to_hashed + ``` -## List attachments on Legacy storage +- **Source installation** -To have a simple summary of project attachments using **Legacy** storage: + ```shell + sudo -u git -H bundle exec rake gitlab:storage:migrate_to_hashed RAILS_ENV=production + ``` -**Omnibus Installation** +If you have any existing integration, you may want to do a small rollout first, +to validate. You can do so by specifying an ID range with the operation by using +the environment variables `ID_FROM` and `ID_TO`. For example, to limit the rollout +to project IDs 50 to 100 in an Omnibus GitLab installation: ```shell -sudo gitlab-rake gitlab:storage:legacy_attachments +sudo gitlab-rake gitlab:storage:migrate_to_hashed ID_FROM=50 ID_TO=100 ``` -**Source Installation** +You can monitor the progress in the **{admin}** **Admin Area > Monitoring > Background Jobs** page. +There is a specific queue you can watch to see how long it will take to finish: +`hashed_storage:hashed_storage_project_migrate`. -```shell -sudo -u git -H bundle exec rake gitlab:storage:legacy_attachments RAILS_ENV=production -``` - -To list project attachments using **Legacy** storage: - -**Omnibus Installation** +After it reaches zero, you can confirm every project has been migrated by running the commands bellow. +If you find it necessary, you can run this migration script again to schedule missing projects. -```shell -sudo gitlab-rake gitlab:storage:list_legacy_attachments -``` +Any error or warning will be logged in Sidekiq's log file. -**Source Installation** +NOTE: **Note:** +If [Geo](../geo/replication/index.md) is enabled, each project that is successfully migrated +generates an event to replicate the changes on any **secondary** nodes. -```shell -sudo -u git -H bundle exec rake gitlab:storage:list_legacy_attachments RAILS_ENV=production -``` +You only need the `gitlab:storage:migrate_to_hashed` Rake task to migrate your repositories, but we have additional +commands below that helps you inspect projects and attachments in both legacy and hashed storage. -## List attachments on Hashed storage +## Rollback from hashed storage to legacy storage -To have a simple summary of project attachments using **Hashed** storage: +NOTE: **Deprecated:** +In GitLab 13.0, [hashed storage](../repository_storage_types.md#hashed-storage) +is enabled by default and the legacy storage is deprecated. +Support for legacy storage will be removed in GitLab 14.0. If you're on GitLab +13.0 and later, switching new projects to legacy storage is not possible. +The option to choose between hashed and legacy storage in the admin area has +been disabled. -**Omnibus Installation** +This task will schedule all your existing projects and associated attachments to be rolled back to the +legacy storage type. -```shell -sudo gitlab-rake gitlab:storage:hashed_attachments -``` +- **Omnibus installation** -**Source Installation** + ```shell + sudo gitlab-rake gitlab:storage:rollback_to_legacy + ``` -```shell -sudo -u git -H bundle exec rake gitlab:storage:hashed_attachments RAILS_ENV=production -``` +- **Source installation** -To list project attachments using **Hashed** storage: + ```shell + sudo -u git -H bundle exec rake gitlab:storage:rollback_to_legacy RAILS_ENV=production + ``` -**Omnibus Installation** +If you have any existing integration, you may want to do a small rollback first, +to validate. You can do so by specifying an ID range with the operation by using +the environment variables `ID_FROM` and `ID_TO`. For example, to limit the rollout +to project IDs 50 to 100 in an Omnibus GitLab installation: ```shell -sudo gitlab-rake gitlab:storage:list_hashed_attachments +sudo gitlab-rake gitlab:storage:rollback_to_legacy ID_FROM=50 ID_TO=100 ``` -**Source Installation** +You can monitor the progress in the **{admin}** **Admin Area > Monitoring > Background Jobs** page. +On the **Queues** tab, you can watch the `hashed_storage:hashed_storage_project_rollback` queue to see how long the process will take to finish. -```shell -sudo -u git -H bundle exec rake gitlab:storage:list_hashed_attachments RAILS_ENV=production -``` +After it reaches zero, you can confirm every project has been rolled back by running the commands bellow. +If some projects weren't rolled back, you can run this rollback script again to schedule further rollbacks. +Any error or warning will be logged in Sidekiq's log file. -[storage-types]: ../repository_storage_types.md -[storage-migration]: ../repository_storage_types.md#how-to-migrate-to-hashed-storage +If you have a Geo setup, the rollback will not be reflected automatically +on the **secondary** node. You may need to wait for a backfill operation to kick-in and remove +the remaining repositories from the special `@hashed/` folder manually. diff --git a/doc/administration/raketasks/uploads/migrate.md b/doc/administration/raketasks/uploads/migrate.md index 851305d433f..d58b802b024 100644 --- a/doc/administration/raketasks/uploads/migrate.md +++ b/doc/administration/raketasks/uploads/migrate.md @@ -1,21 +1,26 @@ -# Uploads Migrate Rake Tasks +# Uploads migrate Rake tasks **(CORE ONLY)** -## Migrate to Object Storage +`gitlab:uploads:migrate` migrates uploads between different storage types. -After [configuring the object storage](../../uploads.md#using-object-storage-core-only) for GitLab's uploads, you may use this task to migrate existing uploads from the local storage to the remote storage. +## Migrate to object storage ->**Note:** -All of the processing will be done in a background worker and requires **no downtime**. +After [configuring the object storage](../../uploads.md#using-object-storage-core-only) for GitLab's +uploads, use this task to migrate existing uploads from the local storage to the remote storage. + +Read more about using [object storage with GitLab](../../object_storage.md). -[Read more about using object storage with GitLab](../../object_storage.md). +NOTE: **Note:** +All of the processing will be done in a background worker and requires **no downtime**. ### All-in-one Rake task -GitLab provides a wrapper Rake task that migrates all uploaded files - avatars, -logos, attachments, favicon, etc. - to object storage in one go. Under the hood, -it invokes individual Rake tasks to migrate files falling under each of this -category one by one. The specifications of these individual Rake tasks are -described in the next section. +GitLab provides a wrapper Rake task that migrates all uploaded files (for example avatars, logos, +attachments, and favicon) to object storage in one step. The wrapper task invokes individual Rake +tasks to migrate files falling under each of these categories one by one. + +These [individual Rake tasks](#individual-rake-tasks) are described in the next section. + +To migrate all uploads from local storage to object storage, run: **Omnibus Installation** @@ -31,26 +36,29 @@ sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:migrate:all ### Individual Rake tasks ->**Note:** -If you already ran the Rake task mentioned above, no need to run these individual Rake tasks as that has been done automatically. +If you already ran the [all-in-one Rake task](#all-in-one-rake-task), there is no need to run these +individual tasks. + +The Rake task uses three parameters to find uploads to migrate: -The Rake task uses 3 parameters to find uploads to migrate. +| Parameter | Type | Description | +|:-----------------|:--------------|:-------------------------------------------------------| +| `uploader_class` | string | Type of the uploader to migrate from. | +| `model_class` | string | Type of the model to migrate from. | +| `mount_point` | string/symbol | Name of the model's column the uploader is mounted on. | -Parameter | Type | Description ---------- | ---- | ----------- -`uploader_class` | string | Type of the uploader to migrate from -`model_class` | string | Type of the model to migrate from -`mount_point` | string/symbol | Name of the model's column on which the uploader is mounted on. +NOTE: **Note:** +These parameters are mainly internal to GitLab's structure, you may want to refer to the task list +instead below. ->**Note:** -These parameters are mainly internal to GitLab's structure, you may want to refer to the task list instead below. +This task also accepts an environment variable which you can use to override +the default batch size: -This task also accepts some environment variables which you can use to override -certain values: +| Variable | Type | Description | +|:---------|:--------|:--------------------------------------------------| +| `BATCH` | integer | Specifies the size of the batch. Defaults to 200. | -Variable | Type | Description --------- | ---- | ----------- -`BATCH` | integer | Specifies the size of the batch. Defaults to 200. +The following shows how to run `gitlab:uploads:migrate` for individual types of uploads. **Omnibus Installation** @@ -76,13 +84,13 @@ gitlab-rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]" gitlab-rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]" gitlab-rake "gitlab:uploads:migrate[FileUploader, MergeRequest]" -# Design Management design thumbnails (EE) +# Design Management design thumbnails gitlab-rake "gitlab:uploads:migrate[DesignManagement::DesignV432x230Uploader, DesignManagement::Action, :image_v432x230]" ``` **Source Installation** ->**Note:** +NOTE: **Note:** Use `RAILS_ENV=production` for every task. ```shell @@ -107,16 +115,14 @@ sudo -u git -H bundle exec rake "gitlab:uploads:migrate[PersonalFileUploader, Sn sudo -u git -H bundle exec rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]" sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, MergeRequest]" -# Design Management design thumbnails (EE) +# Design Management design thumbnails sudo -u git -H bundle exec rake "gitlab:uploads:migrate[DesignManagement::DesignV432x230Uploader, DesignManagement::Action]" ``` -## Migrate from object storage to local storage +## Migrate to local storage -If you need to disable Object Storage for any reason, first you need to migrate -your data out of Object Storage and back into your local storage. - -**Before proceeding, it is important to disable both `direct_upload` and `background_upload` under `uploads` settings in `gitlab.rb`** +If you need to disable [object storage](../../object_storage.md) for any reason, you must first +migrate your data out of object storage and back into your local storage. CAUTION: **Warning:** **Extended downtime is required** so no new files are created in object storage during @@ -126,23 +132,29 @@ To follow progress, see the [relevant issue](https://gitlab.com/gitlab-org/gitla ### All-in-one Rake task -GitLab provides a wrapper Rake task that migrates all uploaded files - avatars, -logos, attachments, favicon, etc. - to local storage in one go. Under the hood, -it invokes individual Rake tasks to migrate files falling under each of this -category one by one. For details on these Rake tasks please [refer to the section above](#individual-rake-tasks), +GitLab provides a wrapper Rake task that migrates all uploaded files (for example, avatars, logos, +attachments, and favicon) to local storage in one step. The wrapper task invokes individual Rake +tasks to migrate files falling under each of these categories one by one. + +For details on these Rake tasks, refer to [Individual Rake tasks](#individual-rake-tasks), keeping in mind the task name in this case is `gitlab:uploads:migrate_to_local`. -**Omnibus Installation** +To migrate uploads from object storage to local storage: -```shell -gitlab-rake "gitlab:uploads:migrate_to_local:all" -``` +1. Disable both `direct_upload` and `background_upload` under `uploads` settings in `gitlab.rb`. +1. Run the Rake task: -**Source Installation** + **Omnibus Installation** -```shell -sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:migrate_to_local:all -``` + ```shell + gitlab-rake "gitlab:uploads:migrate_to_local:all" + ``` + + **Source Installation** + + ```shell + sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:migrate_to_local:all + ``` -After this is done, you may disable Object Storage by undoing the changes described -in the instructions to [configure object storage](../../uploads.md#using-object-storage-core-only) +After running the Rake task, you can disable object storage by undoing the changes described +in the instructions to [configure object storage](../../uploads.md#using-object-storage-core-only). diff --git a/doc/administration/raketasks/uploads/sanitize.md b/doc/administration/raketasks/uploads/sanitize.md index c00399a27cf..7c760b95c33 100644 --- a/doc/administration/raketasks/uploads/sanitize.md +++ b/doc/administration/raketasks/uploads/sanitize.md @@ -1,8 +1,13 @@ -# Uploads Sanitize tasks +# Uploads sanitize Rake tasks **(CORE ONLY)** + +Since GitLab 11.9, EXIF data is automatically stripped from JPG or TIFF image uploads. + +EXIF data may contain sensitive information (for example, GPS location), so you +can remove EXIF data from existing images that were uploaded to an earlier version of GitLab. ## Requirements -You need `exiftool` installed on your system. If you installed GitLab: +To run this Rake task, you need `exiftool` installed on your system. If you installed GitLab: - Using the Omnibus package, you're all set. - From source, make sure `exiftool` is installed: @@ -17,48 +22,48 @@ You need `exiftool` installed on your system. If you installed GitLab: ## Remove EXIF data from existing uploads -Since 11.9 EXIF data are automatically stripped from JPG or TIFF image uploads. -Because EXIF data may contain sensitive information (e.g. GPS location), you -can remove EXIF data also from existing images which were uploaded before -with the following command: +To remove EXIF data from existing uploads, run the following command: ```shell sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif ``` -This command by default runs in dry mode and it doesn't remove EXIF data. It can be used for +By default, this command runs in "dry run" mode and doesn't remove EXIF data. It can be used for checking if (and how many) images should be sanitized. The Rake task accepts following parameters. -Parameter | Type | Description ---------- | ---- | ----------- -`start_id` | integer | Only uploads with equal or greater ID will be processed -`stop_id` | integer | Only uploads with equal or smaller ID will be processed -`dry_run` | boolean | Do not remove EXIF data, only check if EXIF data are present or not, default: true -`sleep_time` | float | Pause for number of seconds after processing each image, default: 0.3 seconds -`uploader` | string | Run sanitization only for uploads of the given uploader (`FileUploader`, `PersonalFileUploader`, `NamespaceFileUploader`) -`since` | date | Run sanitization only for uploads newer than given date (e.g. `2019-05-01`) +| Parameter | Type | Description | +|:-------------|:--------|:----------------------------------------------------------------------------------------------------------------------------| +| `start_id` | integer | Only uploads with equal or greater ID will be processed | +| `stop_id` | integer | Only uploads with equal or smaller ID will be processed | +| `dry_run` | boolean | Do not remove EXIF data, only check if EXIF data are present or not. Defaults to `true` | +| `sleep_time` | float | Pause for number of seconds after processing each image. Defaults to 0.3 seconds | +| `uploader` | string | Run sanitization only for uploads of the given uploader: `FileUploader`, `PersonalFileUploader`, or `NamespaceFileUploader` | +| `since` | date | Run sanitization only for uploads newer than given date. For example, `2019-05-01` | -If you have too many uploads, you can speed up sanitization by setting -`sleep_time` to a lower value or by running multiple Rake tasks in parallel, -each with a separate range of upload IDs (by setting `start_id` and `stop_id`). +If you have too many uploads, you can speed up sanitization by: -To run the command without dry mode and remove EXIF data from all uploads, you can use: +- Setting `sleep_time` to a lower value. +- Running multiple Rake tasks in parallel, each with a separate range of upload IDs (by setting + `start_id` and `stop_id`). + +To remove EXIF data from all uploads, use: ```shell sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif[,,false,] 2>&1 | tee exif.log ``` -To run the command without dry mode on uploads with ID between 100 and 5000 and pause for 0.1 second, you can use: +To remove EXIF data on uploads with an ID between 100 and 5000 and pause for 0.1 second after each file, use: ```shell sudo RAILS_ENV=production -u git -H bundle exec rake gitlab:uploads:sanitize:remove_exif[100,5000,false,0.1] 2>&1 | tee exif.log ``` -Because the output of commands will be probably long, the output is written also into exif.log file. +The output is written into an `exif.log` file because it will probably be long. + +If sanitization fails for an upload, an error message should be in the output of the Rake task. +Typical reasons include that the file is missing in the storage or it's not a valid image. -If sanitization fails for an upload, an error message should be in the output of the Rake task (typical reasons may -be that the file is missing in the storage or it's not a valid image). Please -[report](https://gitlab.com/gitlab-org/gitlab-foss/issues/new) any issues at `gitlab.com` and use -prefix 'EXIF' in issue title with the error output and (if possible) the image. +[Report](https://gitlab.com/gitlab-org/gitlab/issues/new) any issues and use the prefix 'EXIF' in +the issue title with the error output and (if possible) the image. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md new file mode 100644 index 00000000000..7f31f336251 --- /dev/null +++ b/doc/administration/reference_architectures/10k_users.md @@ -0,0 +1,79 @@ +# Reference architecture: up to 10,000 users + +This page describes GitLab reference architecture for up to 10,000 users. +For a full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). + +> - **Supported users (approximate):** 10,000 +> - **High Availability:** True +> - **Test RPS rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|------------------------| +| GitLab Rails ([1](#footnotes)) | 3 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| PostgreSQL | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | +| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Object Storage ([4](#footnotes)) | - | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md new file mode 100644 index 00000000000..b6aaffa9488 --- /dev/null +++ b/doc/administration/reference_architectures/1k_users.md @@ -0,0 +1,82 @@ +# Reference architecture: up to 1,000 users + +This page describes GitLab reference architecture for up to 1,000 users. +For a full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). + +> - **Supported users (approximate):** 1,000 +> - **High Availability:** False + +| Users | Configuration([8](#footnotes)) | GCP | AWS([9](#footnotes)) | Azure([9](#footnotes)) | +|-------|--------------------------------|---------------|----------------------|------------------------| +| 100 | 2 vCPU, 7.2GB Memory | n1-standard-2 | m5.large | D2s v3 | +| 500 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| 1000 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | D8s v3 | + +For situations where you need to serve up to 1,000 users, a single-node +solution with [frequent backups](index.md#automated-backups-core-only) is appropriate +for many organizations. With automatic backup of the GitLab repositories, +configuration, and the database, if you don't have strict availability +requirements, this is the ideal solution. + +## Setup instructions + +- For this default reference architecture, use the standard [installation instructions](../../install/README.md) to install GitLab. + +NOTE: **Note:** +You can also optionally configure GitLab to use an +[external PostgreSQL service](../external_database.md) or an +[external object storage service](../high_availability/object_storage.md) for +added performance and reliability at a reduced complexity cost. + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md new file mode 100644 index 00000000000..2ee692d635c --- /dev/null +++ b/doc/administration/reference_architectures/25k_users.md @@ -0,0 +1,79 @@ +# Reference architecture: up to 25,000 users + +This page describes GitLab reference architecture for up to 25,000 users. +For a full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). + +> - **Supported users (approximate):** 25,000 +> - **High Availability:** True +> - **Test RPS rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|------------------------| +| GitLab Rails ([1](#footnotes)) | 5 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| PostgreSQL | 3 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 32 vCPU, 120GB Memory | n1-standard-32 | m5.8xlarge | D32s v3 | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | +| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Object Storage ([4](#footnotes)) | - | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md new file mode 100644 index 00000000000..874e00e6722 --- /dev/null +++ b/doc/administration/reference_architectures/2k_users.md @@ -0,0 +1,90 @@ +# Reference architecture: up to 2,000 users + +This page describes GitLab reference architecture for up to 2,000 users. +For a full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). + +> - **Supported users (approximate):** 2,000 +> - **High Availability:** False +> - **Test RPS rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|----------------| +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Object Storage ([4](#footnotes)) | - | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| PostgreSQL | 1 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| Redis ([3](#footnotes)) | 1 | 1 vCPU, 3.75GB Memory | n1-standard-1 | m5.large | D2s v3 | +| Gitaly ([5](#footnotes)) ([7](#footnotes)) | X ([2](#footnotes)) | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails ([1](#footnotes)) | 2 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | + +## Setup instructions + +1. [Configure the external load balancing node](../high_availability/load_balancer.md) + that will handle the load balancing of the two GitLab application services nodes. +1. [Configure the Object Storage](../object_storage.md) ([4](#footnotes)) used for shared data objects. +1. (Optional) [Configure NFS](../high_availability/nfs.md) to have + shared disk storage service as an alternative to Gitaly and/or + [Object Storage](../object_storage.md) (although not recommended). + NFS is required for GitLab Pages, you can skip this step if you're not using that feature. +1. [Configure PostgreSQL](../high_availability/load_balancer.md), the database for GitLab. +1. [Configure Redis](../high_availability/redis.md). +1. [Configure Gitaly](../gitaly/index.md#running-gitaly-on-its-own-server), + which is used to provide access to the Git repositories. +1. [Configure the main GitLab Rails application](../high_availability/gitlab.md) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all + frontend requests (UI, API, Git over HTTP/SSH). +1. [Configure Prometheus](../high_availability/monitoring_node.md) to monitor your GitLab environment. + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md new file mode 100644 index 00000000000..bd429fbc4b4 --- /dev/null +++ b/doc/administration/reference_architectures/3k_users.md @@ -0,0 +1,82 @@ +# Reference architecture: up to 3,000 users + +This page describes GitLab reference architecture for up to 3,000 users. +For a full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). + +NOTE: **Note:** The 3,000-user reference architecture documented below is +designed to help your organization achieve a highly-available GitLab deployment. +If you do not have the expertise or need to maintain a highly-available +environment, you can have a simpler and less costly-to-operate environment by +following the [2,000-user reference architecture](2k_users.md). + +> - **Supported users (approximate):** 3,000 +> - **High Availability:** True +> - **Test RPS rates:** API: 60 RPS, Web: 6 RPS, Git: 6 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|------------------------| +| GitLab Rails ([1](#footnotes)) | 3 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| Object Storage ([4](#footnotes)) | - | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md new file mode 100644 index 00000000000..67f773a021f --- /dev/null +++ b/doc/administration/reference_architectures/50k_users.md @@ -0,0 +1,79 @@ +# Reference architecture: up to 50,000 users + +This page describes GitLab reference architecture for up to 50,000 users. +For a full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). + +> - **Supported users (approximate):** 50,000 +> - **High Availability:** True +> - **Test RPS rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|------------------------| +| GitLab Rails ([1](#footnotes)) | 12 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | +| PostgreSQL | 3 | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | D16s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 64 vCPU, 240GB Memory | n1-standard-64 | m5.16xlarge | D64s v3 | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | +| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Object Storage ([4](#footnotes)) | - | - | - | - | - | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md new file mode 100644 index 00000000000..41ef6f369c2 --- /dev/null +++ b/doc/administration/reference_architectures/5k_users.md @@ -0,0 +1,76 @@ +# Reference architecture: up to 5,000 users + +This page describes GitLab reference architecture for up to 5,000 users. +For a full list of reference architectures, see +[Available reference architectures](index.md#available-reference-architectures). + +> - **Supported users (approximate):** 5,000 +> - **High Availability:** True +> - **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|------------------------| +| GitLab Rails ([1](#footnotes)) | 3 | 16 vCPU, 14.4GB Memory | n1-highcpu-16 | c5.4xlarge | F16s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| Object Storage ([4](#footnotes)) | - | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/img/reference-architectures.png b/doc/administration/reference_architectures/img/reference-architectures.png Binary files differnew file mode 100644 index 00000000000..e15609e78e1 --- /dev/null +++ b/doc/administration/reference_architectures/img/reference-architectures.png diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md new file mode 100644 index 00000000000..26244368234 --- /dev/null +++ b/doc/administration/reference_architectures/index.md @@ -0,0 +1,225 @@ +--- +type: reference, concepts +--- +# Reference architectures + +<!-- TBD to be reviewed by Eric --> + +You can set up GitLab on a single server or scale it up to serve many users. +This page details the recommended Reference Architectures that were built and verified by GitLab's Quality and Support teams. + +Below is a chart representing each architecture tier and the number of users they can handle. As your number of users grow with time, it’s recommended that you scale GitLab accordingly. + + +<!-- Internal link: https://docs.google.com/spreadsheets/d/1obYP4fLKkVVDOljaI3-ozhmCiPtEeMblbBKkf2OADKs/edit#gid=1403207183 --> + +Testing on these reference architectures were performed with [GitLab's Performance Tool](https://gitlab.com/gitlab-org/quality/performance) +at specific coded workloads, and the throughputs used for testing were calculated based on sample customer data. +After selecting the reference architecture that matches your scale, refer to +[Configure GitLab to Scale](#configure-gitlab-to-scale) to see the components +involved, and how to configure them. + +Each endpoint type is tested with the following number of requests per second (RPS) per 1000 users: + +- API: 20 RPS +- Web: 2 RPS +- Git: 2 RPS + +For GitLab instances with less than 2,000 users, it's recommended that you use the [default setup](#automated-backups-core-only) +by [installing GitLab](../../install/README.md) on a single machine to minimize maintenance and resource costs. + +If your organization has more than 2,000 users, the recommendation is to scale GitLab's components to multiple +machine nodes. The machine nodes are grouped by component(s). The addition of these +nodes increases the performance and scalability of to your GitLab instance. + +When scaling GitLab, there are several factors to consider: + +- Multiple application nodes to handle frontend traffic. +- A load balancer is added in front to distribute traffic across the application nodes. +- The application nodes connects to a shared file server and PostgreSQL and Redis services on the backend. + +NOTE: **Note:** Depending on your workflow, the following recommended +reference architectures may need to be adapted accordingly. Your workload +is influenced by factors including how active your users are, +how much automation you use, mirroring, and repository/change size. Additionally the +displayed memory values are provided by [GCP machine types](https://cloud.google.com/compute/docs/machine-types). +For different cloud vendors, attempt to select options that best match the provided architecture. + +## Available reference architectures + +The following reference architectures are available: + +- [Up to 1,000 users](1k_users.md) +- [Up to 2,000 users](2k_users.md) +- [Up to 3,000 users](3k_users.md) +- [Up to 5,000 users](5k_users.md) +- [Up to 10,000 users](10k_users.md) +- [Up to 25,000 users](25k_users.md) +- [Up to 50,000 users](50k_users.md) + +## Availability Components + +GitLab comes with the following components for your use, listed from +least to most complex: + +1. [Automated backups](#automated-backups-core-only) +1. [Traffic load balancer](#traffic-load-balancer-starter-only) +1. [Zero downtime updates](#zero-downtime-updates-starter-only) +1. [Automated database failover](#automated-database-failover-premium-only) +1. [Instance level replication with GitLab Geo](#instance-level-replication-with-gitlab-geo-premium-only) + +As you implement these components, begin with a single server and then do +backups. Only after completing the first server should you proceed to the next. + +Also, not implementing extra servers for GitLab doesn't necessarily mean that you'll have +more downtime. Depending on your needs and experience level, single servers can +have more actual perceived uptime for your users. + +### Automated backups **(CORE ONLY)** + +> - Level of complexity: **Low** +> - Required domain knowledge: PostgreSQL, GitLab configurations, Git +> - Supported tiers: [GitLab Core, Starter, Premium, and Ultimate](https://about.gitlab.com/pricing/) + +This solution is appropriate for many teams that have the default GitLab installation. +With automatic backups of the GitLab repositories, configuration, and the database, +this can be an optimal solution if you don't have strict requirements. +[Automated backups](../../raketasks/backup_restore.md#configuring-cron-to-make-daily-backups) +is the least complex to setup. This provides a point-in-time recovery of a predetermined schedule. + +### Traffic load balancer **(STARTER ONLY)** + +> - Level of complexity: **Medium** +> - Required domain knowledge: HAProxy, shared storage, distributed systems +> - Supported tiers: [GitLab Starter, Premium, and Ultimate](https://about.gitlab.com/pricing/) + +This requires separating out GitLab into multiple application nodes with an added +[load balancer](../high_availability/load_balancer.md). The load balancer will distribute traffic +across GitLab application nodes. Meanwhile, each application node connects to a +shared file server and database systems on the back end. This way, if one of the +application servers fails, the workflow is not interrupted. +[HAProxy](https://www.haproxy.org/) is recommended as the load balancer. + +With this added component you have a number of advantages compared +to the default installation: + +- Increase the number of users. +- Enable zero-downtime upgrades. +- Increase availability. + +### Zero downtime updates **(STARTER ONLY)** + +> - Level of complexity: **Medium** +> - Required domain knowledge: PostgreSQL, HAProxy, shared storage, distributed systems +> - Supported tiers: [GitLab Starter, Premium, and Ultimate](https://about.gitlab.com/pricing/) + +GitLab supports [zero-downtime updates](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates). +Although you can perform zero-downtime updates with a single GitLab node, the recommendation is to separate GitLab into several application nodes. +As long as at least one of each component is online and capable of handling the instance's usage load, your team's productivity will not be interrupted during the update. + +### Automated database failover **(PREMIUM ONLY)** + +> - Level of complexity: **High** +> - Required domain knowledge: PgBouncer, Repmgr, shared storage, distributed systems +> - Supported tiers: [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/) + +By adding automatic failover for database systems, you can enable higher uptime +with additional database nodes. This extends the default database with +cluster management and failover policies. +[PgBouncer](../../development/architecture.md#pgbouncer) in conjunction with +[Repmgr](../high_availability/database.md) is recommended. + +### Instance level replication with GitLab Geo **(PREMIUM ONLY)** + +> - Level of complexity: **Very High** +> - Required domain knowledge: Storage replication +> - Supported tiers: [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/) + +[GitLab Geo](../geo/replication/index.md) allows you to replicate your GitLab +instance to other geographical locations as a read-only fully operational instance +that can also be promoted in case of disaster. + +## Configure GitLab to scale + +The following components are the ones you need to configure in order to scale +GitLab. They are listed in the order you'll typically configure them if they are +required by your [reference architecture](#reference-architectures) of choice. + +Most of them are bundled in the GitLab deb/rpm package (called Omnibus GitLab), +but depending on your system architecture, you may require some components which are +not included in it. If required, those should be configured before +setting up components provided by GitLab. Advice on how to select the right +solution for your organization is provided in the configuration instructions +column. + +| Component | Description | Configuration instructions | Bundled with Omnibus GitLab | +|-----------|-------------|----------------------------| +| Load balancer(s) ([6](#footnotes)) | Handles load balancing, typically when you have multiple GitLab application services nodes | [Load balancer configuration](../high_availability/load_balancer.md) ([6](#footnotes)) | No | +| Object storage service ([4](#footnotes)) | Recommended store for shared data objects | [Object Storage configuration](../object_storage.md) | No | +| NFS ([5](#footnotes)) ([7](#footnotes)) | Shared disk storage service. Can be used as an alternative Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | No | +| [Consul](../../development/architecture.md#consul) ([3](#footnotes)) | Service discovery and health checks/failover | [Consul configuration](../high_availability/consul.md) **(PREMIUM ONLY)** | Yes | +| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | Yes | +| [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | Yes | +| Repmgr | PostgreSQL cluster management and failover | [PostgreSQL and Repmgr configuration](../high_availability/database.md) | Yes | +| [Redis](../../development/architecture.md#redis) ([3](#footnotes)) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | Yes | +| Redis Sentinel | Redis | [Redis Sentinel configuration](../high_availability/redis.md) | Yes | +| [Gitaly](../../development/architecture.md#gitaly) ([2](#footnotes)) ([7](#footnotes)) ([10](#footnotes)) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#running-gitaly-on-its-own-server) | Yes | +| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | Yes | +| [GitLab application services](../../development/architecture.md#unicorn)([1](#footnotes)) | Puma/Unicorn, Workhorse, GitLab Shell - serves front-end requests (UI, API, Git over HTTP/SSH) | [GitLab app scaling configuration](../high_availability/gitlab.md) | Yes | +| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling](../high_availability/monitoring_node.md) | Yes | + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. For + nodes that are running Rails with other components the worker value should be reduced + accordingly where we've found 50% achieves a good balance but this is dependent + on workload. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum, + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 3,000 users) a single instance should suffice. + For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) + over NFS where possible, due to better performance. + +1. NFS can be used as an alternative for object storage but this isn't typically + recommended for performance reasons. Note however it is required for [GitLab + Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent and Azure-equivalent configurations are rough suggestions + and may change in the future. They have not yet been tested and validated. + +1. From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab + 14.0, support for NFS for Git repositories is scheduled to be removed. + Upgrade to [Gitaly Cluster](../gitaly/praefect.md) as soon as possible. diff --git a/doc/administration/reply_by_email_postfix_setup.md b/doc/administration/reply_by_email_postfix_setup.md index 8a24514aec2..86854a2a7b6 100644 --- a/doc/administration/reply_by_email_postfix_setup.md +++ b/doc/administration/reply_by_email_postfix_setup.md @@ -1,7 +1,7 @@ # Set up Postfix for incoming email This document will take you through the steps of setting up a basic Postfix mail -server with IMAP authentication on Ubuntu, to be used with [incoming email]. +server with IMAP authentication on Ubuntu, to be used with [incoming email](incoming_email.md). The instructions make the assumption that you will be using the email address `incoming@gitlab.example.com`, that is, username `incoming` on host `gitlab.example.com`. Don't forget to change it to your actual host when executing the example code snippets. @@ -333,10 +333,8 @@ Courier, which we will install later to add IMAP authentication, requires mailbo ## Done -If all the tests were successful, Postfix is all set up and ready to receive email! Continue with the [incoming email] guide to configure GitLab. +If all the tests were successful, Postfix is all set up and ready to receive email! Continue with the [incoming email](incoming_email.md) guide to configure GitLab. --- _This document was adapted from <https://help.ubuntu.com/community/PostfixBasicSetupHowto>, by contributors to the Ubuntu documentation wiki._ - -[incoming email]: incoming_email.md diff --git a/doc/administration/repository_checks.md b/doc/administration/repository_checks.md index a647bc82660..6d9ab723d2f 100644 --- a/doc/administration/repository_checks.md +++ b/doc/administration/repository_checks.md @@ -1,14 +1,14 @@ # Repository checks -> [Introduced][ce-3232] in GitLab 8.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3232) in GitLab 8.7. -Git has a built-in mechanism, [`git fsck`][git-fsck], to verify the +Git has a built-in mechanism, [`git fsck`](https://git-scm.com/docs/git-fsck), to verify the integrity of all data committed to a repository. GitLab administrators can trigger such a check for a project via the project page under the admin panel. The checks run asynchronously so it may take a few minutes before the check result is visible on the project admin page. If the -checks failed you can see their output on the admin log page under -'repocheck.log'. +checks failed you can see their output on in the [`repocheck.log` +file.](logs.md#repochecklog) NOTE: **Note:** It is OFF by default because it still causes too many false alarms. @@ -31,17 +31,11 @@ panel. ## What to do if a check failed If the repository check fails for some repository you should look up the error -in `repocheck.log`: +in the [`repocheck.log` file](logs.md#repochecklog) on disk: -- in the [admin panel](logs.md#repochecklog) -- or on disk, see: - - `/var/log/gitlab/gitlab-rails` for Omnibus installations - - `/home/git/gitlab/log` for installations from source +- `/var/log/gitlab/gitlab-rails` for Omnibus installations +- `/home/git/gitlab/log` for installations from source If the periodic repository check causes false alarms, you can clear all repository check states by navigating to **Admin Area > Settings > Repository** (`/admin/application_settings/repository`) and clicking **Clear all repository checks**. - ---- -[ce-3232]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3232 "Auto git fsck" -[git-fsck]: https://git-scm.com/docs/git-fsck "git fsck documentation" diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index 857fb0b6a90..283401dafff 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -1,6 +1,6 @@ # Repository storage paths -> [Introduced][ce-4578] in GitLab 8.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4578) in GitLab 8.10. GitLab allows you to define multiple repository storage paths (sometimes called storage shards) to distribute the storage load between several mount points. @@ -34,7 +34,7 @@ storage2: ## Configure GitLab > **Warning:** -> In order for [backups] to work correctly, the storage path must **not** be a +> In order for [backups](../raketasks/backup_restore.md) to work correctly, the storage path must **not** be a > mount point and the GitLab user should have correct permissions for the parent > directory of the path. In Omnibus GitLab this is taken care of automatically, > but for source installations you should be extra careful. @@ -47,7 +47,7 @@ storage2: > `gitlab.yml`. > > This little detail matters because while restoring a backup, the current -> contents of `/home/git/repositories` [are moved to][raketask] `/home/git/repositories.old`, +> contents of `/home/git/repositories` [are moved to](https://gitlab.com/gitlab-org/gitlab/blob/033e5423a2594e08a7ebcd2379bd2331f4c39032/lib/backup/repository.rb#L54-56) `/home/git/repositories.old`, > so if `/home/git/repositories` is the mount point, then `mv` would be moving > things between mount points, and bad things could happen. Ideally, > `/home/git` would be the mount point, so then things would be moving within the @@ -79,10 +79,10 @@ NOTE: **Note:** This example uses NFS. We do not recommend using EFS for storage path: /mnt/nfs2/repositories ``` -1. [Restart GitLab][restart-gitlab] for the changes to take effect. +1. [Restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. >**Note:** -The [`gitlab_shell: repos_path` entry][repospath] in `gitlab.yml` will be +The [`gitlab_shell: repos_path` entry](https://gitlab.com/gitlab-org/gitlab-foss/-/blob/8-9-stable/config/gitlab.yml.example#L457) in `gitlab.yml` will be deprecated and replaced by `repositories: storages` in the future, so if you are upgrading from a version prior to 8.10, make sure to add the configuration as described in the step above. After you make the changes and confirm they are @@ -114,9 +114,3 @@ Repository storage > Storage nodes for new repositories**. Beginning with GitLab 8.13.4, multiple paths can be chosen. New repositories will be randomly placed on one of the selected paths. - -[ce-4578]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4578 -[restart-gitlab]: restart_gitlab.md#installations-from-source -[backups]: ../raketasks/backup_restore.md -[raketask]: https://gitlab.com/gitlab-org/gitlab/blob/033e5423a2594e08a7ebcd2379bd2331f4c39032/lib/backup/repository.rb#L54-56 -[repospath]: https://gitlab.com/gitlab-org/gitlab-foss/-/blob/8-9-stable/config/gitlab.yml.example#L457 diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 2e2ed431c8b..a95178c01e2 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -1,16 +1,15 @@ -# Repository Storage Types +# Repository storage types **(CORE ONLY)** -> [Introduced][ce-28283] in GitLab 10.0. - -Two different storage layouts can be used -to store the repositories on disk and their characteristics. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28283) in GitLab 10.0. +> - Hashed storage became the default for new installations in GitLab 12.0 +> - Hashed storage is enabled by default for new and renamed projects in GitLab 13.0. GitLab can be configured to use one or multiple repository storage paths/shard locations that can be: - Mounted to the local disk - Exposed as an NFS shared volume -- Accessed via [Gitaly] on its own machine. +- Accessed via [Gitaly](gitaly/index.md) on its own machine. In GitLab, this is configured in `/etc/gitlab/gitlab.rb` by the `git_data_dirs({})` configuration hash. The storage layouts discussed here will apply to any shard @@ -20,40 +19,17 @@ The `default` repository shard that is available in any installations that haven't customized it, points to the local folder: `/var/opt/gitlab/git-data`. Anything discussed below is expected to be part of that folder. -## Legacy Storage - -Legacy Storage is the storage behavior prior to version 10.0. For historical -reasons, GitLab replicated the same mapping structure from the projects URLs: +## Hashed storage -- Project's repository: `#{namespace}/#{project_name}.git` -- Project's wiki: `#{namespace}/#{project_name}.wiki.git` +NOTE: **Note:** +In GitLab 13.0, hashed storage is enabled by default and the legacy storage is +deprecated. Support for legacy storage will be removed in GitLab 14.0. +If you haven't migrated yet, check the +[migration instructions](raketasks/storage.md#migrate-to-hashed-storage). +The option to choose between hashed and legacy storage in the admin area has +been disabled. -This structure made it simple to migrate from existing solutions to GitLab and -easy for Administrators to find where the repository is stored. - -On the other hand this has some drawbacks: - -Storage location will concentrate huge amount of top-level namespaces. The -impact can be reduced by the introduction of -[multiple storage paths](repository_storage_paths.md). - -Because backups are a snapshot of the same URL mapping, if you try to recover a -very old backup, you need to verify whether any project has taken the place of -an old removed or renamed project sharing the same URL. This means that -`mygroup/myproject` from your backup may not be the same original project that -is at that same URL today. - -Any change in the URL will need to be reflected on disk (when groups / users or -projects are renamed). This can add a lot of load in big installations, -especially if using any type of network based filesystem. - -## Hashed Storage - -CAUTION: **Important:** -Geo requires Hashed Storage since 12.0. If you haven't migrated yet, -check the [migration instructions](#how-to-migrate-to-hashed-storage) ASAP. - -Hashed Storage is the new storage behavior we rolled out with 10.0. Instead +Hashed storage is the storage behavior we rolled out with 10.0. Instead of coupling project URL and the folder structure where the repository will be stored on disk, we are coupling a hash, based on the project's ID. This makes the folder structure immutable, and therefore eliminates any requirement to @@ -124,7 +100,7 @@ GitLab server. For example, on a default Omnibus installation this would be `/var/opt/gitlab/git-data/repositories/@hashed/b1/7e/b17ef6d19c7a5b1ee83b907c595526dcb1eb06db8227d650d5dda0a9f4ce8cd9.git` with `.git` from the end of the directory name removed. -The output includes the project id and the project name: +The output includes the project ID and the project name: ```plaintext => #<Project id:16 it/supportteam/ticketsystem> @@ -134,6 +110,11 @@ The output includes the project id and the project name: > [Introduced](https://gitlab.com/gitlab-org/gitaly/issues/1606) in GitLab 12.1. +DANGER: **Danger:** +Do not run `git prune` or `git gc` in pool repositories! This can +cause data loss in "real" repositories that depend on the pool in +question. + Forks of public projects are deduplicated by creating a third repository, the object pool, containing the objects from the source project. Using `objects/info/alternates`, the source project and forks use the object pool for @@ -145,71 +126,15 @@ when housekeeping is run on the source project. "@pools/#{hash[0..1]}/#{hash[2..3]}/#{hash}.git" ``` -DANGER: **Danger:** -Do not run `git prune` or `git gc` in pool repositories! This can -cause data loss in "real" repositories that depend on the pool in -question. - -### How to migrate to Hashed Storage - -To start a migration, enable Hashed Storage for new projects: - -1. Go to **Admin > Settings > Repository** and expand the **Repository Storage** section. -1. Select the **Use hashed storage paths for newly created and renamed projects** checkbox. - -Check if the change breaks any existing integration you may have that -either runs on the same machine as your repositories are located, or may login to that machine -to access data (for example, a remote backup solution). - -To schedule a complete rollout, see the -[Rake task documentation for storage migration][rake/migrate-to-hashed] for instructions. - -If you do have any existing integration, you may want to do a small rollout first, -to validate. You can do so by specifying a range with the operation. - -This is an example of how to limit the rollout to Project IDs 50 to 100, running in -an Omnibus GitLab installation: - -```shell -sudo gitlab-rake gitlab:storage:migrate_to_hashed ID_FROM=50 ID_TO=100 -``` - -Check the [documentation][rake/migrate-to-hashed] for additional information and instructions for -source-based installation. - -#### Rollback - -Similar to the migration, to disable Hashed Storage for new -projects: - -1. Go to **Admin > Settings > Repository** and expand the **Repository Storage** section. -1. Uncheck the **Use hashed storage paths for newly created and renamed projects** checkbox. - -To schedule a complete rollback, see the -[Rake task documentation for storage rollback](raketasks/storage.md#rollback-from-hashed-storage-to-legacy-storage) for instructions. - -The rollback task also supports specifying a range of Project IDs. Here is an example -of limiting the rollout to Project IDs 50 to 100, in an Omnibus GitLab installation: - -```shell -sudo gitlab-rake gitlab:storage:rollback_to_legacy ID_FROM=50 ID_TO=100 -``` +### Hashed storage coverage migration -If you have a Geo setup, please note that the rollback will not be reflected automatically -on the **secondary** node. You may need to wait for a backfill operation to kick-in and remove -the remaining repositories from the special `@hashed/` folder manually. - -### Hashed Storage coverage - -We are incrementally moving every storable object in GitLab to the Hashed -Storage pattern. You can check the current coverage status below (and also see -the [issue][ce-2821]). - -Note that things stored in an S3 compatible endpoint will not have the downsides +Files stored in an S3 compatible endpoint will not have the downsides mentioned earlier, if they are not prefixed with `#{namespace}/#{project_name}`, which is true for CI Cache and LFS Objects. -| Storable Object | Legacy Storage | Hashed Storage | S3 Compatible | GitLab Version | +In the table below, you can find the coverage of the migration to the hashed storage. + +| Storable Object | Legacy storage | Hashed storage | S3 Compatible | GitLab Version | | --------------- | -------------- | -------------- | ------------- | -------------- | | Repository | Yes | Yes | - | 10.0 | | Attachments | Yes | Yes | - | 10.2 | @@ -222,18 +147,16 @@ which is true for CI Cache and LFS Objects. | LFS Objects | Yes | Similar | Yes | 10.0 / 10.7 | | Repository pools| No | Yes | - | 11.6 | -#### Implementation Details - -##### Avatars +#### Avatars Each file is stored in a folder with its `id` from the database. The filename is always `avatar.png` for user avatars. When avatar is replaced, `Upload` model is destroyed and a new one takes place with different `id`. -##### CI Artifacts +#### CI artifacts CI Artifacts are S3 compatible since **9.4** (GitLab Premium), and available in GitLab Core since **10.6**. -##### LFS Objects +#### LFS objects [LFS Objects in GitLab](../topics/git/lfs/index.md) implement a similar storage pattern using 2 chars, 2 level folders, following Git's own implementation: @@ -247,7 +170,38 @@ storage pattern using 2 chars, 2 level folders, following Git's own implementati LFS objects are also [S3 compatible](lfs/index.md#storing-lfs-objects-in-remote-object-storage). -[ce-2821]: https://gitlab.com/gitlab-com/infrastructure/issues/2821 -[ce-28283]: https://gitlab.com/gitlab-org/gitlab-foss/issues/28283 -[rake/migrate-to-hashed]: raketasks/storage.md#migrate-existing-projects-to-hashed-storage -[gitaly]: gitaly/index.md +## Legacy storage + +NOTE: **Deprecated:** +In GitLab 13.0, hashed storage is enabled by default and the legacy storage is +deprecated. If you haven't migrated yet, check the +[migration instructions](raketasks/storage.md#migrate-to-hashed-storage). +Support for legacy storage will be removed in GitLab 14.0. If you're on GitLab +13.0 and later, switching new projects to legacy storage is not possible. +The option to choose between hashed and legacy storage in the admin area has +been disabled. + +Legacy storage is the storage behavior prior to version 10.0. For historical +reasons, GitLab replicated the same mapping structure from the projects URLs: + +- Project's repository: `#{namespace}/#{project_name}.git` +- Project's wiki: `#{namespace}/#{project_name}.wiki.git` + +This structure made it simple to migrate from existing solutions to GitLab and +easy for Administrators to find where the repository is stored. + +On the other hand this has some drawbacks: + +Storage location will concentrate huge amount of top-level namespaces. The +impact can be reduced by the introduction of +[multiple storage paths](repository_storage_paths.md). + +Because backups are a snapshot of the same URL mapping, if you try to recover a +very old backup, you need to verify whether any project has taken the place of +an old removed or renamed project sharing the same URL. This means that +`mygroup/myproject` from your backup may not be the same original project that +is at that same URL today. + +Any change in the URL will need to be reflected on disk (when groups / users or +projects are renamed). This can add a lot of load in big installations, +especially if using any type of network based filesystem. diff --git a/doc/administration/restart_gitlab.md b/doc/administration/restart_gitlab.md index 907d7bb307a..57f53fd6cc4 100644 --- a/doc/administration/restart_gitlab.md +++ b/doc/administration/restart_gitlab.md @@ -12,18 +12,18 @@ If you want the TL;DR versions, jump to: ## Omnibus installations -If you have used the [Omnibus packages][omnibus-dl] to install GitLab, then +If you have used the [Omnibus packages](https://about.gitlab.com/install/) to install GitLab, then you should already have `gitlab-ctl` in your `PATH`. `gitlab-ctl` interacts with the Omnibus packages and can be used to restart the -GitLab Rails application (Unicorn) as well as the other components, like: +GitLab Rails application (Puma) as well as the other components, like: - GitLab Workhorse - Sidekiq - PostgreSQL (if you are using the bundled one) - NGINX (if you are using the bundled one) - Redis (if you are using the bundled one) -- [Mailroom][] +- [Mailroom](reply_by_email.md) - Logrotate ### Omnibus GitLab restart @@ -45,7 +45,7 @@ ok: run: nginx: (pid 11309) 0s ok: run: postgresql: (pid 11316) 1s ok: run: redis: (pid 11325) 0s ok: run: sidekiq: (pid 11331) 1s -ok: run: unicorn: (pid 11338) 0s +ok: run: puma: (pid 11338) 0s ``` To restart a component separately, you can append its service name to the @@ -86,7 +86,7 @@ sudo gitlab-ctl reconfigure Reconfiguring GitLab should occur in the event that something in its configuration (`/etc/gitlab/gitlab.rb`) has changed. -When you run this command, [Chef], the underlying configuration management +When you run this command, [Chef](https://www.chef.io/products/chef-infra/), the underlying configuration management application that powers Omnibus GitLab, will make sure that all things like directories, permissions, and services are in place and in the same shape that they were initially shipped. @@ -101,7 +101,7 @@ depend on those files. ## Installations from source If you have followed the official installation guide to [install GitLab from -source][install], run the following command to restart GitLab: +source](../install/installation.md), run the following command to restart GitLab: ```shell sudo service gitlab restart @@ -110,46 +110,39 @@ sudo service gitlab restart The output should be similar to this: ```plaintext -Shutting down GitLab Unicorn +Shutting down GitLab Puma Shutting down GitLab Sidekiq Shutting down GitLab Workhorse Shutting down GitLab MailRoom ... GitLab is not running. -Starting GitLab Unicorn +Starting GitLab Puma Starting GitLab Sidekiq Starting GitLab Workhorse Starting GitLab MailRoom ... -The GitLab Unicorn web server with pid 28059 is running. +The GitLab Puma web server with pid 28059 is running. The GitLab Sidekiq job dispatcher with pid 28176 is running. The GitLab Workhorse with pid 28122 is running. The GitLab MailRoom email processor with pid 28114 is running. GitLab and all its components are up and running. ``` -This should restart Unicorn, Sidekiq, GitLab Workhorse, and [Mailroom][] +This should restart Puma, Sidekiq, GitLab Workhorse, and [Mailroom](reply_by_email.md) (if enabled). The init service file that does all the magic can be found on your server in `/etc/init.d/gitlab`. --- If you are using other init systems, like systemd, you can check the -[GitLab Recipes][gl-recipes] repository for some unofficial services. These are +[GitLab Recipes](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/init) repository for some unofficial services. These are **not** officially supported so use them at your own risk. -[omnibus-dl]: https://about.gitlab.com/install/ "Download the Omnibus packages" -[install]: ../install/installation.md "Documentation to install GitLab from source" -[mailroom]: reply_by_email.md "Used for replying by email in GitLab issues and merge requests" -[chef]: https://www.chef.io/products/chef-infra/ "Chef official website" -[src-service]: https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/init.d/gitlab "GitLab init service file" -[gl-recipes]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/init "GitLab Recipes repository" - ## Helm chart installations There is no single command to restart the entire GitLab application installed via the [cloud native Helm Chart](https://docs.gitlab.com/charts/). Usually, it should be -enough to restart a specific component separately (for example, `gitaly`, `unicorn`, +enough to restart a specific component separately (for example, `gitaly`, `puma`, `workhorse`, or `gitlab-shell`) by deleting all the pods related to it: ```shell diff --git a/doc/administration/scaling/index.md b/doc/administration/scaling/index.md index ec7492883cc..748373c8941 100644 --- a/doc/administration/scaling/index.md +++ b/doc/administration/scaling/index.md @@ -1,256 +1,5 @@ --- -type: reference, concepts +redirect_to: ../reference_architectures/index.md --- -# Scaling - -GitLab supports a number of scaling options to ensure that your self-managed -instance is able to scale out to meet your organization's needs when scaling up -a single-box GitLab installation is no longer practical or feasible. - -Please consult our [high availability documentation](../availability/index.md) -if your organization requires fault tolerance and redundancy features, such as -automatic database system failover. - -## GitLab components and scaling instructions - -Here's a list of components directly provided by Omnibus GitLab or installed as -part of a source installation and their configuration instructions for scaling. - -| Component | Description | Configuration instructions | -|-----------|-------------|----------------------------| -| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | -| [Redis](../../development/architecture.md#redis) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | -| [GitLab application services](../../development/architecture.md#unicorn) | Unicorn/Puma, Workhorse, GitLab Shell - serves front-end requests (UI, API, Git over HTTP/SSH) | [GitLab app scaling configuration](../high_availability/gitlab.md) | -| [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | -| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | -| [Gitaly](../../development/architecture.md#gitaly) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#running-gitaly-on-its-own-server) | -| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling](../high_availability/monitoring_node.md) | - -## Third-party services used for scaling - -Here's a list of third-party services you may require as part of scaling GitLab. -The services can be provided by numerous applications or vendors and further -advice is given on how best to select the right choice for your organization's -needs. - -| Component | Description | Configuration instructions | -|-----------|-------------|----------------------------| -| Load balancer(s) | Handles load balancing, typically when you have multiple GitLab application services nodes | [Load balancer configuration](../high_availability/load_balancer.md) | -| Object storage service | Recommended store for shared data objects | [Cloud Object Storage configuration](../high_availability/object_storage.md) | -| NFS | Shared disk storage service. Can be used as an alternative for Gitaly or Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | - -## Reference architectures - -- 1 - 1000 Users: A single-node [Omnibus](https://docs.gitlab.com/omnibus/) setup with frequent backups. Refer to the [Single-node Omnibus installation](#single-node-installation) section below. -- 1000 to 50000+ Users: A [Scaled-out Omnibus installation with multiple servers](#multi-node-installation-scaled-out-for-availability), it can be with or without high-availability components applied. - - To decide the level of Availability please refer to our [Availability](../availability/index.md) page. - -### Single-node installation - -This solution is appropriate for many teams that have a single server at their disposal. With automatic backup of the GitLab repositories, configuration, and the database, this can be an optimal solution if you don't have strict availability requirements. - -You can also optionally configure GitLab to use an [external PostgreSQL service](../external_database.md) -or an [external object storage service](../high_availability/object_storage.md) for added -performance and reliability at a relatively low complexity cost. - -References: - -- [Installation Docs](../../install/README.md) -- [Backup/Restore Docs](https://docs.gitlab.com/omnibus/settings/backups.html#backup-and-restore-omnibus-gitlab-configuration) - -### Multi-node installation (scaled out for availability) - -This solution is appropriate for teams that are starting to scale out when -scaling up is no longer meeting their needs. In this configuration, additional application nodes will handle frontend traffic, with a load balancer in front to distribute traffic across those nodes. Meanwhile, each application node connects to a shared file server and PostgreSQL and Redis services on the back end. - -The additional application servers adds limited fault tolerance to your GitLab -instance. As long as one application node is online and capable of handling the -instance's usage load, your team's productivity will not be interrupted. Having -multiple application nodes also enables [zero-downtime updates](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates). - -References: - -- [Configure your load balancer for GitLab](../high_availability/load_balancer.md) -- [Configure your NFS server to work with GitLab](../high_availability/nfs.md) -- [Configure packaged PostgreSQL server to listen on TCP/IP](https://docs.gitlab.com/omnibus/settings/database.html#configure-packaged-postgresql-server-to-listen-on-tcpip) -- [Setting up a Redis-only server](https://docs.gitlab.com/omnibus/settings/redis.html#setting-up-a-redis-only-server) - -In this section we'll detail the Reference Architectures that can support large numbers -of users. These were built, tested and verified by our Quality and Support teams. - -Testing was done with our GitLab Performance Tool at specific coded workloads, and the -throughputs used for testing were calculated based on sample customer data. We -test each endpoint type with the following number of requests per second (RPS) -per 1000 users: - -- API: 20 RPS -- Web: 2 RPS -- Git: 2 RPS - -NOTE: **Note:** Note that depending on your workflow the below recommended -reference architectures may need to be adapted accordingly. Your workload -is influenced by factors such as - but not limited to - how active your users are, -how much automation you use, mirroring, and repo/change size. Additionally the -shown memory values are given directly by [GCP machine types](https://cloud.google.com/compute/docs/machine-types). -On different cloud vendors a best effort like for like can be used. - -#### 2,000 user configuration - -- **Supported users (approximate):** 2,000 -- **Test RPS rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration[^8] | GCP type | AWS type[^9] | -| ----------------------------|-------|-----------------------|---------------|--------------| -| GitLab Rails[^1] | 3 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly[^2] [^5] [^7] | X | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis[^3] | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Consul + Sentinel[^3] | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Cloud Object Storage[^4] | - | - | - | - | -| NFS Server[^5] [^7] | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | - -#### 5,000 user configuration - -- **Supported users (approximate):** 5,000 -- **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration[^8] | GCP type | AWS type[^9] | -| ----------------------------|-------|------------------------|---------------|--------------| -| GitLab Rails[^1] | 3 | 16 vCPU, 14.4GB Memory | n1-highcpu-16 | c5.4xlarge | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly[^2] [^5] [^7] | X | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | -| Redis[^3] | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Consul + Sentinel[^3] | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Cloud Object Storage[^4] | - | - | - | - | -| NFS Server[^5] [^7] | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | - -#### 10,000 user configuration - -- **Supported users (approximate):** 10,000 -- **Test RPS rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | GCP Configuration[^8] | GCP type | AWS type[^9] | -| ----------------------------|-------|------------------------|----------------|--------------| -| GitLab Rails[^1] | 3 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | -| PostgreSQL | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly[^2] [^5] [^7] | X | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | -| Redis[^3] - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis[^3] - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis Sentinel[^3] - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Redis Sentinel[^3] - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Cloud Object Storage[^4] | - | - | - | - | -| NFS Server[^5] [^7] | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | - -#### 25,000 user configuration - -- **Supported users (approximate):** 25,000 -- **Test RPS rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration[^8] | GCP type | AWS type[^9] | -| ----------------------------|-------|------------------------|----------------|--------------| -| GitLab Rails[^1] | 5 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | -| PostgreSQL | 3 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly[^2] [^5] [^7] | X | 32 vCPU, 120GB Memory | n1-standard-32 | m5.8xlarge | -| Redis[^3] - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis[^3] - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis Sentinel[^3] - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Redis Sentinel[^3] - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Cloud Object Storage[^4] | - | - | - | - | -| NFS Server[^5] [^7] | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node[^6] | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | - -#### 50,000 user configuration - -- **Supported users (approximate):** 50,000 -- **Test RPS rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration[^8] | GCP type | AWS type[^9] | -| ----------------------------|-------|------------------------|----------------|--------------| -| GitLab Rails[^1] | 12 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | -| PostgreSQL | 3 | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly[^2] [^5] [^7] | X | 64 vCPU, 240GB Memory | n1-standard-64 | m5.16xlarge | -| Redis[^3] - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis[^3] - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis Sentinel[^3] - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Redis Sentinel[^3] - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| NFS Server[^5] [^7] | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Cloud Object Storage[^4] | - | - | - | - | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| External load balancing node[^6] | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node[^6] | 1 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | - -[^1]: In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with 4 threads. - -[^2]: Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend 2 nodes as an absolute minimum for HA environments - and at least 4 nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -[^3]: Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (up to 5,000 users) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately as well for each Redis Cluster. - -[^4]: For data objects such as LFS, Uploads, Artifacts, etc. We recommend a [Cloud Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -[^5]: NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). - -[^6]: Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. However other reputable load balancers with similar feature sets - should also work instead but be aware these aren't validated. - -[^7]: We strongly recommend that any Gitaly and / or NFS nodes are set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -[^8]: The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -[^9]: AWS-equivalent configurations are rough suggestions and may change in the - future. They have not yet been tested and validated. +This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 06c560a01ca..0b8c66805ae 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -11,12 +11,13 @@ disqus_identifier: 'https://docs.gitlab.com/ee/administration/custom_hooks.html' > - Server hooks must be configured on the filesystem of the GitLab server. Only GitLab server administrators will be able to complete these tasks. Please explore [webhooks](../user/project/integrations/webhooks.md) and [GitLab CI/CD](../ci/README.md) as an option if you do not have filesystem access. For a user-configurable Git hook interface, see [Push Rules](../push_rules/push_rules.md), available in GitLab Starter **(STARTER)**. > - Server hooks won't be replicated to secondary nodes if you use [GitLab Geo](geo/replication/index.md). -Git natively supports hooks that are executed on different actions. -Examples of server-side Git hooks include pre-receive, post-receive, and update. -See [Git SCM Server-Side Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#Server-Side-Hooks) for more information about each hook type. +Git natively supports hooks that are executed on different actions. These hooks run +on the server and can be used to enforce specific commit policies or perform other +tasks based on the state of the repository. -As of GitLab Shell version 2.2.0 (which requires GitLab 7.5+), GitLab -administrators can add custom Git hooks to any GitLab project. +Examples of server-side Git hooks include `pre-receive`, `post-receive`, and `update`. +See [Git SCM Server-Side Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#Server-Side-Hooks) +for more information about each hook type. ## Create a server hook for a repository @@ -24,14 +25,21 @@ Server-side Git hooks are typically placed in the repository's `hooks` subdirectory. In GitLab, hook directories are symlinked to the GitLab Shell `hooks` directory for ease of maintenance between GitLab Shell upgrades. Server hooks are implemented differently, but the behavior is exactly the same -once the hook is created. Follow the steps below to set up a server hook for a +once the hook is created. + +NOTE: **Note:** +If you are not using [hashed storage](repository_storage_types.md#hashed-storage), the project's +repository directory might not exactly match the instructions below. In that case, +for an installation from source the path is usually `/home/git/repositories/<group>/<project>.git`. +For Omnibus installs the path is usually `/var/opt/gitlab/git-data/repositories/<group>/<project>.git`. + +Follow the steps below to set up a server hook for a repository: -1. Pick a project that needs a server hook. -1. On the GitLab server, navigate to the project's repository directory. - For an installation from source the path is usually - `/home/git/repositories/<group>/<project>.git`. For Omnibus installs the path is - usually `/var/opt/gitlab/git-data/repositories/<group>/<project>.git`. +1. Find that project's path on the GitLab server, by navigating to the + **Admin area > Projects**. From there, select the project for which you + would like to add a hook. You can find the path to the project's repository + under **Gitaly relative path** on that page. 1. Create a new directory in this location called `custom_hooks`. 1. Inside the new `custom_hooks` directory, create a file with a name matching the hook type. For a pre-receive hook the file name should be `pre-receive` @@ -42,8 +50,7 @@ repository: type. For example, if the script is in Ruby the shebang will probably be `#!/usr/bin/env ruby`. -That's it! Assuming the hook code is properly implemented the hook will fire -as appropriate. +Assuming the hook code is properly implemented the hook will run as appropriate. ## Set a global server hook for all repositories diff --git a/doc/administration/smime_signing_email.md b/doc/administration/smime_signing_email.md index ed7447c0da9..bab7c5c260d 100644 --- a/doc/administration/smime_signing_email.md +++ b/doc/administration/smime_signing_email.md @@ -18,6 +18,9 @@ files must be provided: intervention. - Only RSA keys are supported. +Optionally, you can also provide a bundle of CA certs (PEM-encoded) to be +included on each signature. This will typically be an intermediate CA. + NOTE: **Note:** Be mindful of the access levels for your private keys and visibility to third parties. @@ -29,6 +32,8 @@ third parties. gitlab_rails['gitlab_email_smime_enabled'] = true gitlab_rails['gitlab_email_smime_key_file'] = '/etc/gitlab/ssl/gitlab_smime.key' gitlab_rails['gitlab_email_smime_cert_file'] = '/etc/gitlab/ssl/gitlab_smime.crt' + # Optional + gitlab_rails['gitlab_email_smime_ca_certs_file'] = '/etc/gitlab/ssl/gitlab_smime_cas.crt' ``` 1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -49,6 +54,9 @@ NOTE: **Note:** The key needs to be readable by the GitLab system user (`git` by # S/MIME public certificate key in PEM format, will be attached to signed messages # Default is '.gitlab_smime_cert' relative to Rails.root (i.e. root of the GitLab app). cert_file: /etc/pki/smime/certs/gitlab.crt + # S/MIME extra CA public certificates in PEM format, will be attached to signed messages + # Optional + ca_certs_file: /etc/pki/smime/certs/gitlab_cas.crt ``` 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. diff --git a/doc/administration/static_objects_external_storage.md b/doc/administration/static_objects_external_storage.md index f649a1ebcd2..973f4304115 100644 --- a/doc/administration/static_objects_external_storage.md +++ b/doc/administration/static_objects_external_storage.md @@ -63,7 +63,7 @@ other CDNs or Function as a Service (FaaS) systems should work using the same pr `pwgen -cn1 64` on a UNIX machine). Save this token for the admin panel, as described in the [configuring](#configuring) section. - ```js + ```javascript const ORIGIN_HOSTNAME = 'gitlab.installation.com' // FIXME: SET CORRECT VALUE const STORAGE_TOKEN = 'very-secure-token' // FIXME: SET CORRECT VALUE const CACHE_PRIVATE_OBJECTS = false diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 0956edaf252..77d5495418e 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -30,7 +30,7 @@ below. gitlab_rails['terraform_state_storage_path'] = "/mnt/storage/terraform_state" ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. **In installations from source:** @@ -43,7 +43,7 @@ below. storage_path: /mnt/storage/terraform_state ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ## Using object storage **(CORE ONLY)** @@ -62,7 +62,7 @@ The following settings are: | Setting | Description | Default | |---------|-------------|---------| -| `enabled` | Enable/disable object storage | `true` | +| `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where Terraform state files will be stored | | | `connection` | Various connection options described below | | @@ -111,7 +111,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a } ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. **In installations from source:** @@ -131,7 +131,4 @@ The connection settings match those provided by [Fog](https://github.com/fog), a region: eu-central-1 ``` -1. Save the file and [restart GitLab][] for the changes to take effect. - -[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" -[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab" +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. diff --git a/doc/administration/timezone.md b/doc/administration/timezone.md index fa0fbb43433..d6112c45141 100644 --- a/doc/administration/timezone.md +++ b/doc/administration/timezone.md @@ -2,7 +2,7 @@ The global time zone configuration parameter can be changed in `config/gitlab.yml`: -```text +```plaintext # time_zone: 'UTC' ``` @@ -15,7 +15,7 @@ To see all available time zones, run `bundle exec rake time:zones:all`. For Omnibus installations, run `gitlab-rake time:zones:all`. NOTE: **Note:** -Currently, this Rake task does not list timezones in TZInfo format required by GitLab Omnibus during a reconfigure: [#58672](https://gitlab.com/gitlab-org/gitlab-foss/issues/58672). +Currently, this Rake task does not list timezones in TZInfo format required by Omnibus GitLab during a reconfigure: [#58672](https://gitlab.com/gitlab-org/gitlab-foss/issues/58672). ## Changing time zone in Omnibus installations diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index 77a63112ea3..a39fe4ba8c3 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -1,5 +1,8 @@ # Troubleshooting Elasticsearch +To install and configure Elasticsearch, and for common and known issues, +visit the [administrator documentation](../../integration/elasticsearch.md). + Troubleshooting Elasticsearch requires: - Knowledge of common terms. @@ -203,7 +206,7 @@ The best place to start is to determine if the issue is with creating an empty i If it is, check on the Elasticsearch side to determine if the `gitlab-production` (the name for the GitLab index) exists. If it exists, manually delete it on the Elasticsearch side and attempt to recreate it from the -[`create_empty_index`](../../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) +[`recreate_index`](../../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) Rake task. If you still encounter issues, try creating an index manually on the Elasticsearch diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index ba2224e3fc7..2cbc994fb4c 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -150,9 +150,9 @@ Project.update_all(visibility_level: 0) # projects = Project.where(pending_delete: true) projects.each do |p| - puts "Project name: #{p.id}" + puts "Project ID: #{p.id}" puts "Project name: #{p.name}" - puts "Repository path: #{p.repository.storage_path}" + puts "Repository path: #{p.repository.full_path}" end # @@ -214,6 +214,20 @@ p.each do |project| end ``` +### Bulk update to disable the Slack Notification service + +To disable notifications for all projects that have Slack service enabled, do: + +```ruby +# Grab all projects that have the Slack notifications enabled +p = Project.find_by_sql("SELECT p.id FROM projects p LEFT JOIN services s ON p.id = s.project_id WHERE s.type = 'SlackService' AND s.active = true") + +# Disable the service on each of the projects that were found. +p.each do |project| + project.slack_service.update_attribute(:active, false) +end +``` + ## Wikis ### Recreate @@ -558,29 +572,7 @@ Ci::Pipeline.where(project_id: p.id).where(status: 'pending').count ### Remove artifacts more than a week old -The Latest version of these steps can be found in the [job artifacts documentation](../job_artifacts.md) - -```ruby -### SELECTING THE BUILDS TO CLEAR -# For a single project: -project = Project.find_by_full_path('') -builds_with_artifacts = project.builds.with_artifacts_archive - -# Instance-wide: -builds_with_artifacts = Ci::Build.with_artifacts_archive - -# Prior to 10.6 the above lines would be: -# builds_with_artifacts = project.builds.with_artifacts -# builds_with_artifacts = Ci::Build.with_artifacts - -### CLEAR THEM OUT -# Note that this will also erase artifacts that developers marked to "Keep" -builds_to_clear = builds_with_artifacts.where("finished_at < ?", 1.week.ago) -builds_to_clear.each do |build| - build.artifacts_expire_at = Time.now - build.erase_erasable_artifacts! -end -``` +This section has been moved to the [job artifacts troubleshooting documentation](../job_artifacts.md#delete-job-artifacts-from-jobs-completed-before-a-specific-date). ### Find reason failure (for when build trace is empty) (Introduced in 10.3.0) @@ -603,6 +595,14 @@ m = project.merge_requests.find_by(iid: ) m.project.try(:ci_service) ``` +### Validate the `.gitlab-ci.yml` + +```ruby +project = Project.find_by_full_path 'group/project' +content = project.repository.gitlab_ci_yml_for(project.repository.root_ref_sha) +Gitlab::Ci::YamlProcessor.validation_message(content, user: User.first) +``` + ### Disable AutoDevOps on Existing Projects ```ruby diff --git a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md index 30ef3da3a99..cab073b9924 100644 --- a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md +++ b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md @@ -52,7 +52,7 @@ and they will assist you with any issues you are having. - Check logs via Kubectl: ```shell - kubectl logs <unicorn pod> -c dependencies + kubectl logs <webservice pod> -c dependencies ``` - How to tail all Kubernetes cluster events in real time: @@ -72,7 +72,7 @@ and they will assist you with any issues you are having. This is the principle of Kubernetes, read [Twelve-factor app](https://12factor.net/) for details. -- How to get cronjobs configured on a cluster +- How to get cron jobs configured on a cluster ```shell kubectl get cronjobs @@ -87,20 +87,20 @@ and they will assist you with any issues you are having. - Minimal config that can be used to test a Kubernetes Helm chart can be found [here](https://gitlab.com/gitlab-org/charts/gitlab/issues/620). -- Tailing logs of a separate pod. An example for a Unicorn pod: +- Tailing logs of a separate pod. An example for a Webservice pod: ```shell - kubectl logs gitlab-unicorn-7656fdd6bf-jqzfs -c unicorn + kubectl logs gitlab-webservice-54fbf6698b-hpckq -c webservice ``` -- Tail and follow all pods that share a label (in this case, `unicorn`): +- Tail and follow all pods that share a label (in this case, `webservice`): ```shell - # all containers in the unicorn pods - kubectl logs -f -l app=unicorn --all-containers=true --max-log-requests=50 + # all containers in the webservice pods + kubectl logs -f -l app=webservice --all-containers=true --max-log-requests=50 - # only the unicorn containers in all unicorn pods - kubectl logs -f -l app=unicorn -c unicorn --max-log-requests=50 + # only the webservice containers in all webservice pods + kubectl logs -f -l app=webservice -c webservice --max-log-requests=50 ``` - One can stream logs from all containers at once, similar to the Omnibus @@ -132,7 +132,7 @@ and they will assist you with any issues you are having. /srv/gitlab/bin/rails console # source-style commands should also work - /srv/gitlab && bundle exec rake gitlab:check RAILS_ENV=production + cd /srv/gitlab && bundle exec rake gitlab:check RAILS_ENV=production # run GitLab check. Note that the output can be confusing and invalid because of the specific structure of GitLab installed via helm chart /usr/local/bin/gitlab-rake gitlab:check @@ -206,7 +206,7 @@ all Kubernetes resources and dependent charts: helm get manifest <release name> ``` -## Installation of minimal GitLab config via Minukube on macOS +## Installation of minimal GitLab config via Minikube on macOS This section is based on [Developing for Kubernetes with Minikube](https://docs.gitlab.com/charts/development/minikube/index.html) and [Helm](https://docs.gitlab.com/charts/installation/tools.html#helm). Refer @@ -230,31 +230,33 @@ to those documents for details. ```shell minikube start --cpus 3 --memory 8192 # minimum amount for GitLab to work minikube addons enable ingress - minikube addons enable kube-dns ``` - Install Helm via Homebrew and initialize it: ```shell - brew install kubernetes-helm - helm init --service-account tiller + brew install helm ``` -- Copy the file <https://gitlab.com/gitlab-org/charts/gitlab/raw/master/examples/values-minikube-minimum.yaml> - to your workstation. +- Copy the [Minikube minimum values YAML file](https://gitlab.com/gitlab-org/charts/gitlab/raw/master/examples/values-minikube-minimum.yaml) + to your workstation: -- Find the IP address in the output of `minikube ip` and update the yaml file with + ```shell + curl --output values.yaml "https://gitlab.com/gitlab-org/charts/gitlab/raw/master/examples/values-minikube-minimum.yaml" + ``` + +- Find the IP address in the output of `minikube ip` and update the YAML file with this IP address. - Install the GitLab Helm Chart: ```shell helm repo add gitlab https://charts.gitlab.io - helm install --name gitlab -f <path-to-yaml-file> gitlab/gitlab + helm install gitlab -f <path-to-yaml-file> gitlab/gitlab ``` If you want to modify some GitLab settings, you can use the above-mentioned config - as a base and create your own yaml file. + as a base and create your own YAML file. - Monitor the installation progress via `helm status gitlab` and `minikube dashboard`. The installation could take up to 20-30 minutes depending on the amount of resources @@ -263,7 +265,7 @@ to those documents for details. - When all the pods show either a `Running` or `Completed` status, get the GitLab password as described in [Initial login](https://docs.gitlab.com/charts/installation/deployment.html#initial-login), and log in to GitLab via the UI. It will be accessible via `https://gitlab.domain` - where `domain` is the value provided in the yaml file. + where `domain` is the value provided in the YAML file. <!-- ## Troubleshooting diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index 0413e5ce953..dcd1df2f423 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -16,19 +16,19 @@ include use cases targeted for parsing GitLab log files. #### Pipe colorized `jq` output into `less` -```sh +```shell jq . <FILE> -C | less -R ``` #### Search for a term and pretty-print all matching lines -```sh +```shell grep <TERM> <FILE> | jq . ``` #### Skip invalid lines of JSON -```sh +```shell jq -cR 'fromjson?' file.json | jq <COMMAND> ``` @@ -39,49 +39,49 @@ This skips over all invalid lines and parses the rest. #### Find all requests with a 5XX status code -```sh +```shell jq 'select(status >= 500)' <FILE> ``` #### Top 10 slowest requests -```sh +```shell jq -s 'sort_by(-.duration) | limit(10; .[])' <FILE> ``` #### Find and pretty print all requests related to a project -```sh +```shell grep <PROJECT_NAME> <FILE> | jq . ``` #### Find all requests with a total duration > 5 seconds -```sh +```shell jq 'select(.duration > 5000)' <FILE> ``` #### Find all project requests with more than 5 rugged calls -```sh +```shell grep <PROJECT_NAME> <FILE> | jq 'select(.rugged_calls > 5)' ``` #### Find all requests with a Gitaly duration > 10 seconds -```sh +```shell jq 'select(.gitaly_duration > 10000)' <FILE> ``` #### Find all requests with a queue duration > 10 seconds -```sh +```shell jq 'select(.queue_duration > 10000)' <FILE> ``` #### Top 10 requests by # of Gitaly calls -```sh +```shell jq -s 'map(select(.gitaly_calls != null)) | sort_by(-.gitaly_calls) | limit(10; .[])' <FILE> ``` @@ -89,7 +89,7 @@ jq -s 'map(select(.gitaly_calls != null)) | sort_by(-.gitaly_calls) | limit(10; #### Print the top three controller methods by request volume and their three longest durations -```sh +```shell jq -s -r 'group_by(.controller+.action) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration) | "CT: \(length)\tMETHOD: \(.[0].controller)#\(.[0].action)\tDURS: \(.[0].duration), \(.[1].duration), \(.[2].duration)"' production_json.log ``` @@ -105,7 +105,7 @@ CT: 1328 METHOD: Projects::NotesController#index DURS: 403.99, 386.29, 384.3 #### Print top three routes with request count and their three longest durations -```sh +```shell jq -s -r 'group_by(.route) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration) | "CT: \(length)\tROUTE: \(.[0].route)\tDURS: \(.[0].duration), \(.[1].duration), \(.[2].duration)"' api_json.log ``` @@ -121,25 +121,25 @@ CT: 190 ROUTE: /api/:version/projects/:id/repository/commits DURS: 1079.02, #### Find all Gitaly requests sent from web UI -```sh +```shell jq 'select(."grpc.meta.client_name" == "gitlab-web")' current ``` #### Find all failed Gitaly requests -```sh +```shell jq 'select(."grpc.code" != null and ."grpc.code" != "OK")' current ``` #### Find all requests that took longer than 30 seconds -```sh +```shell jq 'select(."grpc.time_ms" > 30000)' current ``` #### Print top three projects by request volume and their three longest durations -```sh +```shell jq -s -r 'map(select(."grpc.request.glProjectPath" != null and ."grpc.request.glProjectPath" != "" and ."grpc.time_ms" != null)) | group_by(."grpc.request.glProjectPath") | sort_by(-length) | limit(3; .[]) | sort_by(-."grpc.time_ms") | "CT: \(length)\tPROJECT: \(.[0]."grpc.request.glProjectPath")\tDURS: \(.[0]."grpc.time_ms"), \(.[1]."grpc.time_ms"), \(.[2]."grpc.time_ms")"' current ``` diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index 8e7727ee214..65a6bffca44 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -33,7 +33,7 @@ This section is for links to information elsewhere in the GitLab documentation. - [More about external PostgreSQL](../external_database.md) -- [Running GEO with external PostgreSQL](../geo/replication/external_database.md) +- [Running Geo with external PostgreSQL](../geo/replication/external_database.md) - [Upgrades when running PostgreSQL configured for HA.](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-gitlab-ha-cluster) @@ -45,7 +45,7 @@ This section is for links to information elsewhere in the GitLab documentation. - Managing Omnibus PostgreSQL versions [from the development docs](https://docs.gitlab.com/omnibus/development/managing-postgresql-versions.html) -- [PostgreSQL scaling and HA](../high_availability/database.md) +- [PostgreSQL scaling](../high_availability/database.md) - including [troubleshooting](../high_availability/database.md#troubleshooting) `gitlab-ctl repmgr-check-master` and PgBouncer errors - [Developer database documentation](../../development/README.md#database-guides) - some of which is absolutely not for production use. Including: @@ -71,7 +71,7 @@ This section is for links to information elsewhere in the GitLab documentation. HINT: Free one or increase max_replication_slots. ``` -- GEO [replication errors](../geo/replication/troubleshooting.md#fixing-replication-errors) including: +- Geo [replication errors](../geo/replication/troubleshooting.md#fixing-replication-errors) including: ```plaintext ERROR: replication slots can only be used if max_replication_slots > 0 @@ -83,11 +83,11 @@ This section is for links to information elsewhere in the GitLab documentation. PANIC: could not write to file ‘pg_xlog/xlogtemp.123’: No space left on device ``` -- [Checking GEO configuration](../geo/replication/troubleshooting.md#checking-configuration) including +- [Checking Geo configuration](../geo/replication/troubleshooting.md#checking-configuration) including - reconfiguring hosts/ports - checking and fixing user/password mappings -- [Common GEO errors](../geo/replication/troubleshooting.md#fixing-common-errors) +- [Common Geo errors](../geo/replication/troubleshooting.md#fixing-common-errors) ## Support topics diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 31e41725834..ca21c038267 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -282,7 +282,8 @@ Commonly, `<condition>` references the job arguments, which depend on the type o For example, `repository_import` has `project_id` as the job argument, while `update_merge_requests` has `project_id, user_id, oldrev, newrev, ref`. -NOTE: **Note:** Arguments need to be referenced by their sequence id using `job.args[<id>]` because `job.args` is a list of all arguments provided to the Sidekiq job. +NOTE: **Note:** +Arguments need to be referenced by their sequence ID using `job.args[<id>]` because `job.args` is a list of all arguments provided to the Sidekiq job. Here are some examples: diff --git a/doc/administration/troubleshooting/ssl.md b/doc/administration/troubleshooting/ssl.md index f230f047ded..e6c081e1eea 100644 --- a/doc/administration/troubleshooting/ssl.md +++ b/doc/administration/troubleshooting/ssl.md @@ -46,6 +46,44 @@ After configuring a GitLab instance with an internal CA certificate, you might n If you have the problems listed above, add your certificate to `/etc/gitlab/trusted-certs` and run `sudo gitlab-ctl reconfigure`. +## X.509 key values mismatch error + +After configuring your instance with a certificate bundle, NGINX may throw the +following error: + +`SSL: error:0B080074:x509 certificate routines:X509_check_private_key:key values mismatch` + +This error means that the server certificate and key you have provided do not +match. You can confirm this by running the following command and comparing the +output: + +```shell +openssl rsa -noout -modulus -in path/to/your/.key | openssl md5 +openssl x509 -noout -modulus -in path/to/your/.crt | openssl md5 +``` + +The following is an example of an md5 output between a matching key and certificate. Note the +matching md5 hashes: + +```shell +$ openssl rsa -noout -modulus -in private.key | openssl md5 +4f49b61b25225abeb7542b29ae20e98c +$ openssl x509 -noout -modulus -in public.crt | openssl md5 +4f49b61b25225abeb7542b29ae20e98c +``` + +This is an opposing output with a non-matching key and certificate which shows different md5 hashes: + +```shell +$ openssl rsa -noout -modulus -in private.key | openssl md5 +d418865077299af27707b1d1fa83cd99 +$ openssl x509 -noout -modulus -in public.crt | openssl md5 +4f49b61b25225abeb7542b29ae20e98c +``` + +If the two outputs differ like the above example, there is a mismatch between the certificate +and key. You should contact the provider of the SSL certificate for further support. + ## Using GitLab Runner with a GitLab instance configured with internal CA certificate or self-signed certificate Besides getting the errors mentioned in diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index f29deba3d40..0a300084342 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -25,7 +25,7 @@ _The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/uploads`._ gitlab_rails['uploads_base_dir'] = "uploads" ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. **In installations from source:** @@ -41,13 +41,13 @@ _The uploads are stored by default in base_dir: uploads ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ## Using object storage **(CORE ONLY)** > **Notes:** > -> - [Introduced][ee-3867] in [GitLab Premium][eep] 10.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3867) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17358) in [GitLab Core](https://about.gitlab.com/pricing/) 10.7. > - Since version 11.1, we support direct_upload to S3. @@ -65,7 +65,7 @@ For source installations the following settings are nested under `uploads:` and |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where Uploads will be stored| | -| `direct_upload` | Set to true to remove Unicorn from the Upload path. Workhorse handles the actual Artifact Upload to Object Storage while Unicorn does minimal processing to keep track of the upload. There is no need for local shared storage. The option may be removed if support for a single storage type for all files is introduced. Read more on [direct upload](../development/uploads.md#direct-upload). | `false` | +| `direct_upload` | Set to true to remove Puma from the Upload path. Workhorse handles the actual Artifact Upload to Object Storage while Puma does minimal processing to keep track of the upload. There is no need for local shared storage. The option may be removed if support for a single storage type for all files is introduced. Read more on [direct upload](../development/uploads.md#direct-upload). | `false` | | `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 (if `direct_upload` is set to `true` it will override `background_upload`) | `true` | | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | @@ -76,16 +76,16 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | Setting | Description | Default | |---------|-------------|---------| -| `provider` | Always `AWS` for compatible hosts | AWS | +| `provider` | Always `AWS` for compatible hosts | `AWS` | | `aws_access_key_id` | AWS credentials, or compatible | | | `aws_secret_access_key` | AWS credentials, or compatible | | -| `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | -| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | +| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | +| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | | `region` | AWS region | us-east-1 | -| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | +| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | `s3.amazonaws.com` | | `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | -| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | -| `use_iam_profile` | Set to true to use IAM profile instead of access keys | false +| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false` | +| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys | false **In Omnibus installations:** @@ -117,7 +117,7 @@ _The uploads are stored by default in } ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate` Rake task](raketasks/uploads/migrate.md). **In installations from source:** @@ -140,7 +140,7 @@ _The uploads are stored by default in region: eu-central-1 ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. 1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate` Rake task](raketasks/uploads/migrate.md). ### Oracle Cloud S3 connection settings @@ -149,8 +149,8 @@ Note that Oracle Cloud S3 must be sure to use the following settings: | Setting | Value | |---------|-------| -| `enable_signature_v4_streaming` | false | -| `path_style` | true | +| `enable_signature_v4_streaming` | `false` | +| `path_style` | `true` | If `enable_signature_v4_streaming` is set to `true`, you may see the following error: @@ -165,7 +165,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | Setting | Description | Default | |---------|-------------|---------| -| `provider` | Always `OpenStack` for compatible hosts | OpenStack | +| `provider` | Always `OpenStack` for compatible hosts | `OpenStack` | | `openstack_username` | OpenStack username | | | `openstack_api_key` | OpenStack API key | | | `openstack_temp_url_key` | OpenStack key for generating temporary urls | | @@ -194,7 +194,7 @@ _The uploads are stored by default in } ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate` Rake task](raketasks/uploads/migrate.md). --- @@ -225,10 +225,5 @@ _The uploads are stored by default in openstack_tenant: 'TENANT_ID' ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate` Rake task](raketasks/uploads/migrate.md). - -[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" -[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab" -[eep]: https://about.gitlab.com/pricing/ "GitLab Premium" -[ee-3867]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3867 diff --git a/doc/api/README.md b/doc/api/README.md index 3c8d3dc4902..34d496a37fe 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -58,7 +58,7 @@ Currently only API version v4 is available. Version v3 was removed in ## Basic usage API requests should be prefixed with `api` and the API version. The API version -is defined in [`lib/api.rb`][lib-api-url]. For example, the root of the v4 API +is defined in [`lib/api.rb`](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/lib/api/api.rb). For example, the root of the v4 API is at `/api/v4`. Example of a valid API request using cURL: @@ -75,12 +75,13 @@ end of an API URL. Most API requests require authentication, or will only return public data when authentication is not provided. For those cases where it is not required, this will be mentioned in the documentation -for each individual endpoint. For example, the [`/projects/:id` endpoint](projects.md). +for each individual endpoint. For example, the [`/projects/:id` endpoint](projects.md#get-single-project). -There are four ways to authenticate with the GitLab API: +There are several ways to authenticate with the GitLab API: 1. [OAuth2 tokens](#oauth2-tokens) -1. [Personal access tokens](#personal-access-tokens) +1. [Personal access tokens](../user/profile/personal_access_tokens.md) +1. [Project access tokens](../user/project/settings/project_access_tokens.md) **(CORE ONLY)** 1. [Session cookie](#session-cookie) 1. [GitLab CI/CD job token](#gitlab-ci-job-token) **(Specific endpoints only)** @@ -117,31 +118,29 @@ curl --header "Authorization: Bearer OAUTH-TOKEN" https://gitlab.example.com/api Read more about [GitLab as an OAuth2 provider](oauth2.md). -### Personal access tokens +### Personal/project access tokens -You can use a [personal access token][pat] to authenticate with the API by passing it in either the -`private_token` parameter or the `Private-Token` header. +Access tokens can be used to authenticate with the API by passing it in either the `private_token` parameter +or the `Private-Token` header. -Example of using the personal access token in a parameter: +Example of using the personal/project access token in a parameter: ```shell curl https://gitlab.example.com/api/v4/projects?private_token=<your_access_token> ``` -Example of using the personal access token in a header: +Example of using the personal/project access token in a header: ```shell curl --header "Private-Token: <your_access_token>" https://gitlab.example.com/api/v4/projects ``` -You can also use personal access tokens with OAuth-compliant headers: +You can also use personal/project access tokens with OAuth-compliant headers: ```shell curl --header "Authorization: Bearer <your_access_token>" https://gitlab.example.com/api/v4/projects ``` -Read more about [personal access tokens][pat]. - ### Session cookie When signing in to the main GitLab application, a `_gitlab_session` cookie is @@ -163,9 +162,9 @@ to authenticate with the API: ### Impersonation tokens -> [Introduced][ce-9099] in GitLab 9.0. Needs admin permissions. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9099) in GitLab 9.0. Needs admin permissions. -Impersonation tokens are a type of [personal access token][pat] +Impersonation tokens are a type of [personal access token](../user/profile/personal_access_tokens.md) that can only be created by an admin for a specific user. They are a great fit if you want to build applications or scripts that authenticate with the API as a specific user. @@ -417,7 +416,7 @@ The response header includes a link to the next page. For example: ```http HTTP/1.1 200 OK ... -Link: <https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc&id_after=42>; rel="next" +Links: <https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc&id_after=42>; rel="next" Status: 200 OK ... ``` @@ -425,7 +424,7 @@ Status: 200 OK The link to the next page contains an additional filter `id_after=42` which excludes records we have retrieved already. Note the type of filter depends on the `order_by` option used and we may have more than one additional filter. -When the end of the collection has been reached and there are no additional records to retrieve, the `Link` header is absent and the resulting array is empty. +When the end of the collection has been reached and there are no additional records to retrieve, the `Links` header is absent and the resulting array is empty. We recommend using only the given link to retrieve the next page instead of building your own URL. Apart from the headers shown, we don't expose additional pagination headers. @@ -446,14 +445,20 @@ For example: DELETE /projects/:id/share/:group_id ``` -The `:id` path parameter needs to be replaced with the project id, and the `:group_id` needs to be replaced with the id of the group. The colons `:` should not be included. +The `:id` path parameter needs to be replaced with the project ID, and the `:group_id` needs to be replaced with the ID of the group. The colons `:` should not be included. -The resulting cURL call for a project with id `5` and a group id of `17` is then: +The resulting cURL call for a project with ID `5` and a group ID of `17` is then: ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/share/17 ``` +NOTE: **Note:** +Path parameters that are required to be URL-encoded must be followed. If not, +it will not match an API endpoint and respond with a 404. If there's something +in front of the API (for example, Apache), ensure that it won't decode the URL-encoded +path parameters. + ## Namespaced path encoding If using namespaced API calls, make sure that the `NAMESPACE/PROJECT_PATH` is @@ -470,15 +475,16 @@ A project's **path** is not necessarily the same as its **name**. A project's path can be found in the project's URL or in the project's settings under **General > Advanced > Change path**. -## Branches and tags name encoding +## File path, branches, and tags name encoding -If your branch or tag contains a `/`, make sure the branch/tag name is -URL-encoded. +If a file path, branch or tag contains a `/`, make sure it is URL-encoded. For example, `/` is represented by `%2F`: ```plaintext +GET /api/v4/projects/1/repository/files/src%2FREADME.md?ref=master GET /api/v4/projects/1/branches/my%2Fbranch/commits +GET /api/v4/projects/1/repository/tags/my%2Ftag ``` ## Encoding API parameters of `array` and `hash` types @@ -642,9 +648,3 @@ For administrator documentation on rate limit settings, see [Rate limits](../security/rate_limits.md). To find the settings that are specifically used by GitLab.com, see [GitLab.com-specific rate limits](../user/gitlab_com/index.md#gitlabcom-specific-rate-limits). - -[lib-api-url]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/lib/api/api.rb -[ce-3749]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3749 -[ce-5951]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5951 -[ce-9099]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9099 -[pat]: ../user/profile/personal_access_tokens.md diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index 8b3d8462cc9..2adf06a8e95 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -26,11 +26,14 @@ The following API resources are available in the project context: | [Custom attributes](custom_attributes.md) | `/projects/:id/custom_attributes` (also available for groups and users) | | [Dependencies](dependencies.md) **(ULTIMATE)** | `/projects/:id/dependencies` | | [Deploy keys](deploy_keys.md) | `/projects/:id/deploy_keys` (also available standalone) | +| [Freeze Periods](freeze_periods.md) | `/projects/:id/freeze_periods` | | [Deployments](deployments.md) | `/projects/:id/deployments` | | [Discussions](discussions.md) (threaded comments) | `/projects/:id/issues/.../discussions`, `/projects/:id/snippets/.../discussions`, `/projects/:id/merge_requests/.../discussions`, `/projects/:id/commits/.../discussions` (also available for groups) | | [Environments](environments.md) | `/projects/:id/environments` | | [Error Tracking](error_tracking.md) | `/projects/:id/error_tracking/settings` | | [Events](events.md) | `/projects/:id/events` (also available for users and standalone) | +| [Feature Flags](feature_flags.md) | `/projects/:id/feature_flags` | +| [Feature Flag User Lists](feature_flag_user_lists.md) | `/projects/:id/feature_flags_user_lists` | | [Issues](issues.md) | `/projects/:id/issues` (also available for groups and standalone) | | [Issues Statistics](issues_statistics.md) | `/projects/:id/issues_statistics` (also available for groups and standalone) | | [Issue boards](boards.md) | `/projects/:id/boards` | @@ -56,7 +59,7 @@ The following API resources are available in the project context: | [Project milestones](milestones.md) | `/projects/:id/milestones` | | [Project snippets](project_snippets.md) | `/projects/:id/snippets` | | [Project templates](project_templates.md) | `/projects/:id/templates` | -| [Protected_environments](protected_environments.md) | `/projects/:id/protected_environments` | +| [Protected environments](protected_environments.md) | `/projects/:id/protected_environments` | | [Protected branches](protected_branches.md) | `/projects/:id/protected_branches` | | [Protected tags](protected_tags.md) | `/projects/:id/protected_tags` | | [Releases](releases/index.md) | `/projects/:id/releases` | @@ -70,7 +73,8 @@ The following API resources are available in the project context: | [Search](search.md) | `/projects/:id/search` (also available for groups and standalone) | | [Services](services.md) | `/projects/:id/services` | | [Tags](tags.md) | `/projects/:id/repository/tags` | -| [Visual Review discussions](visual_review_discussions.md) **(STARTER**) | `/projects/:id/merge_requests/:merge_request_id/visual_review_discussions` | +| [User-starred metrics dashboards](metrics_user_starred_dashboards.md ) | `/projects/:id/metrics/user_starred_dashboards` | +| [Visual Review discussions](visual_review_discussions.md) **(STARTER)** | `/projects/:id/merge_requests/:merge_request_id/visual_review_discussions` | | [Vulnerabilities](vulnerabilities.md) **(ULTIMATE)** | `/vulnerabilities/:id` | | [Vulnerability exports](vulnerability_exports.md) **(ULTIMATE)** | `/projects/:id/vulnerability_exports` | | [Project vulnerabilities](project_vulnerabilities.md) **(ULTIMATE)** | `/projects/:id/vulnerabilities` | @@ -110,7 +114,8 @@ The following API resources are available outside of project and group contexts | Resource | Available endpoints | |:---------------------------------------------------|:------------------------------------------------------------------------| -| [Admin Sidekiq queues](admin_sidekiq_queues.md) | `/admin/sidekiq/queues/:queue_name` | +| [Instance-level CI/CD variables](instance_level_ci_variables.md) | `/admin/ci/variables` | +| [Admin Sidekiq queues](admin_sidekiq_queues.md) | `/admin/sidekiq/queues/:queue_name` | | [Appearance](appearance.md) **(CORE ONLY)** | `/application/appearance` | | [Applications](applications.md) | `/applications` | | [Audit Events](audit_events.md) **(PREMIUM ONLY)** | `/audit_events` | @@ -130,11 +135,12 @@ The following API resources are available outside of project and group contexts | [License](license.md) **(CORE ONLY)** | `/license` | | [Markdown](markdown.md) | `/markdown` | | [Merge requests](merge_requests.md) | `/merge_requests` (also available for groups and projects) | -| [Metrics dashboard annotations](metrics_dashboard_annotations.md) | `/environments/:id/metrics_dashboard/annotations` | +| [Metrics dashboard annotations](metrics_dashboard_annotations.md) | `/environments/:id/metrics_dashboard/annotations`, `/clusters/:id/metrics_dashboard/annotations` | | [Namespaces](namespaces.md) | `/namespaces` | | [Notification settings](notification_settings.md) | `/notification_settings` (also available for groups and projects) | | [Pages domains](pages_domains.md) | `/pages/domains` (also available for projects) | | [Projects](projects.md) | `/users/:id/projects` (also available for projects) | +| [Project Repository Storage Moves](project_repository_storage_moves.md) | `/project_repository_storage_moves` | | [Runners](runners.md) | `/runners` (also available for projects) | | [Search](search.md) | `/search` (also available for groups and projects) | | [Settings](settings.md) **(CORE ONLY)** | `/application/settings` | diff --git a/doc/api/appearance.md b/doc/api/appearance.md index f9ca6aed01a..733d71ee222 100644 --- a/doc/api/appearance.md +++ b/doc/api/appearance.md @@ -27,6 +27,7 @@ Example response: "header_logo": "/uploads/-/system/appearance/header_logo/1/header.png", "favicon": "/uploads/-/system/appearance/favicon/1/favicon.png", "new_project_guidelines": "Please read the FAQs for help.", + "profile_image_guidelines": "Custom profile image guidelines", "header_message": "", "footer_message": "", "message_background_color": "#e75e40", @@ -51,6 +52,7 @@ PUT /application/appearance | `header_logo` | mixed | no | Instance image used for the main navigation bar | `favicon` | mixed | no | Instance favicon in .ico/.png format | `new_project_guidelines` | string | no | Markdown text shown on the new project page +| `profile_image_guidelines` | string | no | Markdown text shown on the profile page below Public Avatar | `header_message` | string | no | Message within the system header bar | `footer_message` | string | no | Message within the system footer bar | `message_background_color` | string | no | Background color for the system header / footer bar @@ -71,6 +73,7 @@ Example response: "header_logo": "/uploads/-/system/appearance/header_logo/1/header.png", "favicon": "/uploads/-/system/appearance/favicon/1/favicon.png", "new_project_guidelines": "Please read the FAQs for help.", + "profile_image_guidelines": "Custom profile image guidelines", "header_message": "test", "footer_message": "", "message_background_color": "#e75e40", diff --git a/doc/api/applications.md b/doc/api/applications.md index c7bfebb75fa..5d4a8b3a99f 100644 --- a/doc/api/applications.md +++ b/doc/api/applications.md @@ -16,7 +16,7 @@ Create an application by posting a JSON payload. Returns `200` if the request succeeds. -```text +```plaintext POST /applications ``` @@ -52,7 +52,7 @@ Example response: List all registered applications. -```text +```plaintext GET /applications ``` @@ -85,7 +85,7 @@ Delete a specific application. Returns `204` if the request succeeds. -```text +```plaintext DELETE /applications/:id ``` @@ -93,7 +93,7 @@ Parameters: | Attribute | Type | Required | Description | |:----------|:--------|:---------|:----------------------------------------------------| -| `id` | integer | yes | The id of the application (not the application_id). | +| `id` | integer | yes | The ID of the application (not the application_id). | Example request: diff --git a/doc/api/avatar.md b/doc/api/avatar.md index 55d5f1e613a..308c0de25f4 100644 --- a/doc/api/avatar.md +++ b/doc/api/avatar.md @@ -15,7 +15,7 @@ If: NOTE: **Note:** This endpoint can be accessed without authentication. -```text +```plaintext GET /avatar?email=admin@example.com ``` diff --git a/doc/api/award_emoji.md b/doc/api/award_emoji.md index 7fb96f4f1c0..37b3cd32f89 100644 --- a/doc/api/award_emoji.md +++ b/doc/api/award_emoji.md @@ -20,7 +20,7 @@ See [Award Emoji on Comments](#award-emoji-on-comments) for information on using Get a list of all award emoji for a specified awardable. -```text +```plaintext GET /projects/:id/issues/:issue_iid/award_emoji GET /projects/:id/merge_requests/:merge_request_iid/award_emoji GET /projects/:id/snippets/:snippet_id/award_emoji @@ -82,7 +82,7 @@ Example response: Get a single award emoji from an issue, snippet, or merge request. -```text +```plaintext GET /projects/:id/issues/:issue_iid/award_emoji/:award_id GET /projects/:id/merge_requests/:merge_request_iid/award_emoji/:award_id GET /projects/:id/snippets/:snippet_id/award_emoji/:award_id @@ -127,7 +127,7 @@ Example response: Create an award emoji on the specified awardable. -```text +```plaintext POST /projects/:id/issues/:issue_iid/award_emoji POST /projects/:id/merge_requests/:merge_request_iid/award_emoji POST /projects/:id/snippets/:snippet_id/award_emoji @@ -173,7 +173,7 @@ Sometimes it's just not meant to be and you'll have to remove the award. NOTE: **Note:** Only available to administrators or the author of the award. -```text +```plaintext DELETE /projects/:id/issues/:issue_iid/award_emoji/:award_id DELETE /projects/:id/merge_requests/:merge_request_iid/award_emoji/:award_id DELETE /projects/:id/snippets/:snippet_id/award_emoji/:award_id @@ -204,7 +204,7 @@ easily adapted for comments on a merge request or on a snippet. Therefore, you h Get all award emoji for a comment (note). -```text +```plaintext GET /projects/:id/issues/:issue_iid/notes/:note_id/award_emoji ``` @@ -249,7 +249,7 @@ Example response: Get a single award emoji for a comment (note). -```text +```plaintext GET /projects/:id/issues/:issue_iid/notes/:note_id/award_emoji/:award_id ``` @@ -293,7 +293,7 @@ Example response: Create an award emoji on the specified comment (note). -```text +```plaintext POST /projects/:id/issues/:issue_iid/notes/:note_id/award_emoji ``` @@ -340,7 +340,7 @@ Sometimes it's just not meant to be and you'll have to remove the award. NOTE: **Note:** Only available to administrators or the author of the award. -```text +```plaintext DELETE /projects/:id/issues/:issue_iid/notes/:note_id/award_emoji/:award_id ``` diff --git a/doc/api/boards.md b/doc/api/boards.md index b99f249cab1..8af23527931 100644 --- a/doc/api/boards.md +++ b/doc/api/boards.md @@ -229,7 +229,7 @@ Example response: ## Update a board **(STARTER)** -> [Introduced][ee-5954] in [GitLab Starter](https://about.gitlab.com/pricing/) 11.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5954) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.1. Updates a board. @@ -511,5 +511,3 @@ DELETE /projects/:id/boards/:board_id/lists/:list_id ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1/lists/1 ``` - -[ee-5954]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5954 diff --git a/doc/api/branches.md b/doc/api/branches.md index 2f9ca62ced6..7d14a3d54b9 100644 --- a/doc/api/branches.md +++ b/doc/api/branches.md @@ -12,7 +12,7 @@ Get a list of repository branches from a project, sorted by name alphabetically. NOTE: **Note:** This endpoint can be accessed without authentication if the repository is publicly accessible. -```text +```plaintext GET /projects/:id/repository/branches ``` @@ -41,6 +41,7 @@ Example response: "developers_can_push": false, "developers_can_merge": false, "can_push": true, + "web_url": "http://gitlab.example.com/my-group/my-project/-/tree/master", "commit": { "author_email": "john@example.com", "author_name": "John Smith", @@ -68,7 +69,7 @@ Get a single project repository branch. NOTE: **Note:** This endpoint can be accessed without authentication if the repository is publicly accessible. -```text +```plaintext GET /projects/:id/repository/branches/:branch ``` @@ -96,6 +97,7 @@ Example response: "developers_can_push": false, "developers_can_merge": false, "can_push": true, + "web_url": "http://gitlab.example.com/my-group/my-project/-/tree/master", "commit": { "author_email": "john@example.com", "author_name": "John Smith", @@ -128,7 +130,7 @@ for information on unprotecting repository branches. Create a new branch in the repository. -```text +```plaintext POST /projects/:id/repository/branches ``` @@ -171,7 +173,8 @@ Example response: "default": false, "developers_can_push": false, "developers_can_merge": false, - "can_push": true + "can_push": true, + "web_url": "http://gitlab.example.com/my-group/my-project/-/tree/newbranch" } ``` @@ -182,7 +185,7 @@ Delete a branch from the repository. NOTE: **Note:** In the case of an error, an explanation message is provided. -```text +```plaintext DELETE /projects/:id/repository/branches/:branch ``` @@ -206,7 +209,7 @@ Will delete all branches that are merged into the project's default branch. NOTE: **Note:** [Protected branches](../user/project/protected_branches.md) will not be deleted as part of this operation. -```text +```plaintext DELETE /projects/:id/repository/merged_branches ``` diff --git a/doc/api/broadcast_messages.md b/doc/api/broadcast_messages.md index 1ff40103750..db2198dc162 100644 --- a/doc/api/broadcast_messages.md +++ b/doc/api/broadcast_messages.md @@ -13,7 +13,7 @@ As of GitLab 12.8, GET requests do not require authentication. All other broadca List all broadcast messages. -```text +```plaintext GET /broadcast_messages ``` @@ -46,7 +46,7 @@ Example response: Get a specific broadcast message. -```text +```plaintext GET /broadcast_messages/:id ``` @@ -83,7 +83,7 @@ Example response: Create a new broadcast message. -```text +```plaintext POST /broadcast_messages ``` @@ -127,7 +127,7 @@ Example response: Update an existing broadcast message. -```text +```plaintext PUT /broadcast_messages/:id ``` diff --git a/doc/api/commits.md b/doc/api/commits.md index 356f090f0ff..98a8e4ea2ce 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -65,7 +65,7 @@ Example response: ## Create a commit with multiple files and actions -> [Introduced][ce-6096] in GitLab 8.13. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6096) in GitLab 8.13. Create a commit by posting a JSON payload @@ -94,7 +94,7 @@ POST /projects/:id/repository/commits | `previous_path` | string | no | Original full path to the file being moved. Ex. `lib/class1.rb`. Only considered for `move` action. | | `content` | string | no | File content, required for all except `delete`, `chmod`, and `move`. Move actions that do not specify `content` will preserve the existing file content, and any other value of `content` will overwrite the file content. | | `encoding` | string | no | `text` or `base64`. `text` is default. | -| `last_commit_id` | string | no | Last known file commit id. Will be only considered in update, move and delete actions. | +| `last_commit_id` | string | no | Last known file commit ID. Will be only considered in update, move, and delete actions. | | `execute_filemode` | boolean | no | When `true/false` enables/disables the execute flag on the file. Only considered for `chmod` action. | ```shell @@ -245,7 +245,7 @@ Example response: ## Get references a commit is pushed to -> [Introduced][ce-15026] in GitLab 10.6 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15026) in GitLab 10.6 Get all references (from branches or tags) a commit is pushed to. The pagination parameters `page` and `per_page` can be used to restrict the list of references. @@ -280,7 +280,7 @@ Example response: ## Cherry pick a commit -> [Introduced][ce-8047] in GitLab 8.15. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8047) in GitLab 8.15. Cherry picks a commit to a given branch. @@ -340,7 +340,7 @@ conflict. ## Revert a commit -> [Introduced][ce-22919] in GitLab 11.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22919) in GitLab 11.5. Reverts a commit in a given branch. @@ -521,6 +521,62 @@ Example response: } ``` +## Get the discussions of a commit + +Get the discussions of a commit in a project. + +```plaintext +GET /projects/:id/repository/commits/:sha/discussions +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user +| `sha` | string | yes | The commit hash or name of a repository branch or tag | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/commits/4604744a1c64de00ff62e1e8a6766919923d2b41/discussions" +``` + +Example response: + +```json +[ + { + "id": "4604744a1c64de00ff62e1e8a6766919923d2b41", + "individual_note": true, + "notes": [ + { + "id": 334686748, + "type": null, + "body": "I'm the Dude, so that's what you call me.", + "attachment": null, + "author" : { + "id" : 28, + "name" : "Jeff Lebowski", + "username" : "thedude", + "web_url" : "https://gitlab.example.com/thedude", + "state" : "active", + "avatar_url" : "https://gitlab.example.com/uploads/user/avatar/28/The-Big-Lebowski-400-400.png" + }, + "created_at": "2020-04-30T18:48:11.432Z", + "updated_at": "2020-04-30T18:48:11.432Z", + "system": false, + "noteable_id": null, + "noteable_type": "Commit", + "resolvable": false, + "confidential": null, + "noteable_iid": null, + "commands_changes": {} + } + ] + } +] + +``` + ## Commit status Since GitLab 8.1, this is the new commit status API. @@ -653,7 +709,7 @@ Example response: ## List Merge Requests associated with a commit -> [Introduced][ce-18004] in GitLab 10.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18004) in GitLab 10.7. Get a list of Merge Requests related to the specified commit. @@ -755,7 +811,7 @@ Example response if commit is GPG signed: } ``` -Example response if commit is x509 signed: +Example response if commit is X.509 signed: ```json { @@ -785,9 +841,3 @@ Example response if commit is unsigned: "message": "404 GPG Signature Not Found" } ``` - -[ce-6096]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6096 "Multi-file commit" -[ce-8047]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8047 -[ce-15026]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15026 -[ce-18004]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18004 -[ce-22919]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22919 diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index a41a71808ce..9ec4373c92c 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -18,6 +18,7 @@ GET /projects/:id/registry/repositories | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) accessible by the authenticated user. | | `tags` | boolean | no | If the parameter is included as true, each repository will include an array of `"tags"` in the response. | +| `name` | string | no | Returns a list of repositories with a name that matches the value. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29763) in GitLab 13.0). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories" @@ -58,6 +59,7 @@ GET /groups/:id/registry/repositories | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) accessible by the authenticated user. | | `tags` | boolean | no | If the parameter is included as true, each repository will include an array of `"tags"` in the response. | +| `name` | string | no | Returns a list of repositories with a name that matches the value. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29763) in GitLab 13.0). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/2/registry/repositories?tags=1" @@ -223,6 +225,9 @@ This action does not delete blobs. In order to delete them and recycle disk spac Delete registry repository tags in bulk based on given criteria. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Utilize the Container Registry API to delete all tags except *](https://youtu.be/Hi19bKe_xsg). + ```plaintext DELETE /projects/:id/registry/repositories/:repository_id/tags ``` diff --git a/doc/api/deploy_keys.md b/doc/api/deploy_keys.md index f6d00988c56..a7acc0c2b55 100644 --- a/doc/api/deploy_keys.md +++ b/doc/api/deploy_keys.md @@ -2,7 +2,7 @@ ## List all deploy keys -Get a list of all deploy keys across all projects of the GitLab instance. This endpoint requires admin access. +Get a list of all deploy keys across all projects of the GitLab instance. This endpoint requires admin access and is not available on GitLab.com. ```plaintext GET /deploy_keys diff --git a/doc/api/deploy_tokens.md b/doc/api/deploy_tokens.md index 461957847df..6e732a43da0 100644 --- a/doc/api/deploy_tokens.md +++ b/doc/api/deploy_tokens.md @@ -92,7 +92,7 @@ POST /projects/:id/deploy_tokens | `name` | string | yes | New deploy token's name | | `expires_at` | datetime | no | Expiration date for the deploy token. Does not expire if no value is provided. | | `username` | string | no | Username for deploy token. Default is `gitlab+deploy-token-{n}` | -| `scopes` | array of strings | yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, or `write_registry`. | +| `scopes` | array of strings | yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`. | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data '{"name": "My deploy token", "expires_at": "2021-01-01", "username": "custom-user", "scopes": ["read_repository"]}' "https://gitlab.example.com/api/v4/projects/5/deploy_tokens/" @@ -193,7 +193,7 @@ POST /groups/:id/deploy_tokens | `name` | string | yes | New deploy token's name | | `expires_at` | datetime | no | Expiration date for the deploy token. Does not expire if no value is provided. | | `username` | string | no | Username for deploy token. Default is `gitlab+deploy-token-{n}` | -| `scopes` | array of strings | yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, or `write_registry`. | +| `scopes` | array of strings | yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`. | Example request: diff --git a/doc/api/epics.md b/doc/api/epics.md index bf6a18fcedc..6ca6f04b741 100644 --- a/doc/api/epics.md +++ b/doc/api/epics.md @@ -1,5 +1,8 @@ # Epics API **(PREMIUM)** +> - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.2. +> - Single-level Epics [were moved](https://gitlab.com/gitlab-org/gitlab/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. + Every API call to epic must be authenticated. If a user is not a member of a group and the group is private, a `GET` request on that group will result to a `404` status code. @@ -64,6 +67,7 @@ GET /groups/:id/epics?state=opened | `updated_before` | datetime | no | Return epics updated on or before the given time | | `include_ancestor_groups` | boolean | no | Include epics from the requested group's ancestors. Default is `false` | | `include_descendant_groups` | boolean | no | Include epics from the requested group's descendants. Default is `true` | +| `my_reaction_emoji` | string | no | Return epics reacted by the authenticated user by the given emoji. `None` returns epics not given a reaction. `Any` returns epics given at least one reaction. Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31479)| ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics @@ -246,7 +250,7 @@ POST /groups/:id/epics | `start_date_fixed` | string | no | The fixed start date of an epic (since 11.3) | | `due_date_is_fixed` | boolean | no | Whether due date should be sourced from `due_date_fixed` or from milestones (since 11.3) | | `due_date_fixed` | string | no | The fixed due date of an epic (since 11.3) | -| `parent_id` | integer/string | no | The id of a parent epic (since 11.11) | +| `parent_id` | integer/string | no | The ID of a parent epic (since 11.11) | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics?title=Epic&description=Epic%20description diff --git a/doc/api/events.md b/doc/api/events.md index 431e96b2804..e6cb56f1339 100644 --- a/doc/api/events.md +++ b/doc/api/events.md @@ -62,10 +62,10 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `action` | string | no | Include only events of a particular [action type][action-types] | -| `target_type` | string | no | Include only events of a particular [target type][target-types] | -| `before` | date | no | Include only events created before a particular date. Please see [here for the supported format][date-formatting] | -| `after` | date | no | Include only events created after a particular date. Please see [here for the supported format][date-formatting] | +| `action` | string | no | Include only events of a particular [action type](#action-types) | +| `target_type` | string | no | Include only events of a particular [target type](#target-types) | +| `before` | date | no | Include only events created before a particular date. Please see [here for the supported format](#date-formatting) | +| `after` | date | no | Include only events created after a particular date. Please see [here for the supported format](#date-formatting) | | `scope` | string | no | Include all events across a user's projects. | | `sort` | string | no | Sort events in `asc` or `desc` order by `created_at`. Default is `desc` | @@ -123,7 +123,7 @@ Example response: ### Get user contribution events >**Notes:** -> Documentation was formerly located in the [Users API pages][users-api]. +> Documentation was formerly located in the [Users API pages](users.md). > `read_user` access was introduced in GitLab 11.3. Get the contribution events for the specified user, sorted from newest to oldest. Scope `read_user` or `api` is required. @@ -137,10 +137,10 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer | yes | The ID or Username of the user | -| `action` | string | no | Include only events of a particular [action type][action-types] | -| `target_type` | string | no | Include only events of a particular [target type][target-types] | -| `before` | date | no | Include only events created before a particular date. Please see [here for the supported format][date-formatting] | -| `after` | date | no | Include only events created after a particular date. Please see [here for the supported format][date-formatting] | +| `action` | string | no | Include only events of a particular [action type](#action-types) | +| `target_type` | string | no | Include only events of a particular [target type](#target-types) | +| `before` | date | no | Include only events created before a particular date. Please see [here for the supported format](#date-formatting) | +| `after` | date | no | Include only events created after a particular date. Please see [here for the supported format](#date-formatting) | | `sort` | string | no | Sort events in `asc` or `desc` order by `created_at`. Default is `desc` | ```shell @@ -255,7 +255,7 @@ Example response: ## List a Project's visible events ->**Note:** This endpoint has been around longer than the others. Documentation was formerly located in the [Projects API pages][projects-api]. +>**Note:** This endpoint has been around longer than the others. Documentation was formerly located in the [Projects API pages](projects.md). Get a list of visible events for a particular project. @@ -268,10 +268,10 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `project_id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `action` | string | no | Include only events of a particular [action type][action-types] | -| `target_type` | string | no | Include only events of a particular [target type][target-types] | -| `before` | date | no | Include only events created before a particular date. Please see [here for the supported format][date-formatting] | -| `after` | date | no | Include only events created after a particular date. Please see [here for the supported format][date-formatting] | +| `action` | string | no | Include only events of a particular [action type](#action-types) | +| `target_type` | string | no | Include only events of a particular [target type](#target-types) | +| `before` | date | no | Include only events created before a particular date. Please see [here for the supported format](#date-formatting) | +| `after` | date | no | Include only events created after a particular date. Please see [here for the supported format](#date-formatting) | | `sort` | string | no | Sort events in `asc` or `desc` order by `created_at`. Default is `desc` | Example request: @@ -365,9 +365,3 @@ Example response: } ] ``` - -[target-types]: #target-types "Target Type parameter" -[action-types]: #action-types "Action Type parameter" -[date-formatting]: #date-formatting "Date Formatting guidance" -[projects-api]: projects.md "Projects API pages" -[users-api]: users.md "Users API pages" diff --git a/doc/api/feature_flag_specs.md b/doc/api/feature_flag_specs.md index 442d2c2c2d7..fbe99826b27 100644 --- a/doc/api/feature_flag_specs.md +++ b/doc/api/feature_flag_specs.md @@ -10,7 +10,7 @@ Users with Developer or higher [permissions](../user/permissions.md) can access ## List all effective feature flag specs under the specified environment -Get all effective feature flag specs under the specified [environment](../ci/environments.md). +Get all effective feature flag specs under the specified [environment](../ci/environments/index.md). For instance, there are two specs, `staging` and `production`, for a feature flag. When you pass `production` as a parameter to this endpoint, the system returns @@ -23,7 +23,7 @@ GET /projects/:id/feature_flag_scopes | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `environment` | string | yes | The [environment](../ci/environments.md) name | +| `environment` | string | yes | The [environment](../ci/environments/index.md) name | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flag_scopes?environment=production @@ -155,7 +155,7 @@ POST /projects/:id/feature_flags/:name/scopes | ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The [environment spec](../ci/environments.md#scoping-environments-with-specs) of the feature flag. | +| `environment_scope` | string | yes | The [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | | `active` | boolean | yes | Whether the spec is active. | | `strategies` | json | yes | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | @@ -202,7 +202,7 @@ GET /projects/:id/feature_flags/:name/scopes/:environment_scope | ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments.md#scoping-environments-with-specs) of the feature flag. | +| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/feature_flags/new_live_trace/scopes/production @@ -238,7 +238,7 @@ PUT /projects/:id/feature_flags/:name/scopes/:environment_scope | ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments.md#scoping-environments-with-specs) of the feature flag. | +| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | | `active` | boolean | yes | Whether the spec is active. | | `strategies` | json | yes | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | @@ -284,7 +284,7 @@ DELETE /projects/:id/feature_flags/:name/scopes/:environment_scope | ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments.md#scoping-environments-with-specs) of the feature flag. | +| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production diff --git a/doc/api/feature_flag_user_lists.md b/doc/api/feature_flag_user_lists.md new file mode 100644 index 00000000000..04cad8f2d1d --- /dev/null +++ b/doc/api/feature_flag_user_lists.md @@ -0,0 +1,181 @@ +# Feature flag user lists API **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205409) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. + +API for accessing GitLab Feature Flag User Lists. + +Users with Developer or higher [permissions](../user/permissions.md) can access the Feature Flag User Lists API. + +NOTE: **Note:** +`GET` requests return twenty results at a time because the API results +are [paginated](README.md#pagination). You can change this value. + +## List all feature flag user lists for a project + +Gets all feature flag user lists for the requested project. + +```plaintext +GET /projects/:id/feature_flags_user_lists +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists +``` + +Example response: + +```json +[ + { + "name": "user_list", + "user_xids": "user1,user2", + "id": 1, + "iid": 1, + "project_id": 1, + "created_at": "2020-02-04T08:13:51.423Z", + "updated_at": "2020-02-04T08:13:51.423Z" + }, + { + "name": "test_users", + "user_xids": "user3,user4,user5", + "id": 2, + "iid": 2, + "project_id": 1, + "created_at": "2020-02-04T08:13:10.507Z", + "updated_at": "2020-02-04T08:13:10.507Z" + } +] +``` + +## Create a feature flag user list + +Creates a feature flag user list. + +```plaintext +POST /projects/:id/feature_flags_user_lists +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `name` | string | yes | The name of the feature flag. | +| `user_xids` | string | yes | A comma separated list of user IDs. | + +```shell +curl https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + --header "Content-type: application/json" \ + --data @- << EOF +{ + "name": "my_user_list", + "user_xids": "user1,user2,user3" +} +EOF +``` + +Example response: + +```json +{ + "name": "my_user_list", + "user_xids": "user1,user2,user3", + "id": 1, + "iid": 1, + "project_id": 1, + "created_at": "2020-02-04T08:32:27.288Z", + "updated_at": "2020-02-04T08:32:27.288Z" +} +``` + +## Get a feature flag user list + +Gets a feature flag user list. + +```plaintext +GET /projects/:id/feature_flags_user_lists/:iid +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `iid` | integer/string | yes | The internal ID of the project's feature flag user list. | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1 +``` + +Example response: + +```json +{ + "name": "my_user_list", + "user_xids": "123,456", + "id": 1, + "iid": 1, + "project_id": 1, + "created_at": "2020-02-04T08:13:10.507Z", + "updated_at": "2020-02-04T08:13:10.507Z", +} +``` + +## Update a feature flag user list + +Updates a feature flag user list. + +```plaintext +PUT /projects/:id/feature_flags_user_lists/:iid +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `iid` | integer/string | yes | The internal ID of the project's feature flag user list. | +| `name` | string | no | The name of the feature flag. | +| `user_xids` | string | no | A comma separated list of user IDs. | + +```shell +curl https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1 \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + --header "Content-type: application/json" \ + --request PUT \ + --data @- << EOF +{ + "user_xids": "user2,user3,user4" +} +EOF +``` + +Example response: + +```json +{ + "name": "my_user_list", + "user_xids": "user2,user3,user4", + "id": 1, + "iid": 1, + "project_id": 1, + "created_at": "2020-02-04T08:32:27.288Z", + "updated_at": "2020-02-05T09:33:17.179Z" +} +``` + +## Delete feature flag user list + +Deletes a feature flag user list. + +```plaintext +DELETE /projects/:id/feature_flags_user_lists/:iid +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `iid` | integer/string | yes | The internal ID of the project's feature flag user list | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1 +``` diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index f95eb31c84c..3e1b0e05298 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -155,7 +155,7 @@ POST /projects/:id/feature_flags | `name` | string | yes | The name of the feature flag. | | `description` | string | no | The description of the feature flag. | | `scopes` | JSON | no | The [feature flag specs](../user/project/operations/feature_flags.md#define-environment-specs) of the feature flag. | -| `scopes:environment_scope` | string | no | The [environment spec](../ci/environments.md#scoping-environments-with-specs). | +| `scopes:environment_scope` | string | no | The [environment spec](../ci/environments/index.md#scoping-environments-with-specs). | | `scopes:active` | boolean | no | Whether the spec is active. | | `scopes:strategies` | JSON | no | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | diff --git a/doc/api/features.md b/doc/api/features.md index a43f2daa93f..03f1663b987 100644 --- a/doc/api/features.md +++ b/doc/api/features.md @@ -1,5 +1,7 @@ # Features flags API +This API is for managing Flipper-based [feature flags used in development of GitLab](../development/feature_flags/index.md). + All methods require administrator authorization. Notice that currently the API only supports boolean and percentage-of-time gate @@ -32,6 +34,16 @@ Example response: ] }, { + "name": "my_user_feature", + "state": "on", + "gates": [ + { + "key": "percentage_of_actors", + "value": 34 + } + ] + }, + { "name": "new_library", "state": "on", "gates": [ @@ -58,6 +70,7 @@ POST /features/:name | --------- | ---- | -------- | ----------- | | `name` | string | yes | Name of the feature to create or update | | `value` | integer/string | yes | `true` or `false` to enable/disable, or an integer for percentage of time | +| `key` | string | no | `percentage_of_actors` or `percentage_of_time` (default) | | `feature_group` | string | no | A Feature group name | | `user` | string | no | A GitLab username | | `group` | string | no | A GitLab group's path, for example `gitlab-org` | @@ -89,6 +102,37 @@ Example response: } ``` +### Set percentage of actors rollout + +Rollout to percentage of actors. + +```plaintext +POST https://gitlab.example.com/api/v4/features/my_user_feature?private_token=<your_access_token> +Content-Type: application/x-www-form-urlencoded +value=42&key=percentage_of_actors& +``` + +Example response: + +```json +{ + "name": "my_user_feature", + "state": "conditional", + "gates": [ + { + "key": "boolean", + "value": false + }, + { + "key": "percentage_of_actors", + "value": 42 + } + ] +} +``` + +Rolls out the `my_user_feature` to `42%` of actors. + ## Delete a feature Removes a feature gate. Response is equal when the gate exists, or doesn't. diff --git a/doc/api/freeze_periods.md b/doc/api/freeze_periods.md new file mode 100644 index 00000000000..ee5e657c945 --- /dev/null +++ b/doc/api/freeze_periods.md @@ -0,0 +1,168 @@ +# Freeze Periods API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29382) in GitLab 13.0. + +You can use the Freeze Periods API to manipulate GitLab's [Freeze Period](../user/project/releases/index.md#set-a-deploy-freeze) entries. + +## Permissions and security + +Only users with Maintainer [permissions](../user/permissions.md) can +interact with the Freeze Period API endpoints. + +## List Freeze Periods + +Paginated list of Freeze Periods, sorted by `created_at` in ascending order. + +```plaintext +GET /projects/:id/freeze_periods +``` + +| Attribute | Type | Required | Description | +| ------------- | -------------- | -------- | ----------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: gVWYVHDRzXiRpN1rUC8T" "https://gitlab.example.com/api/v4/projects/19/freeze_periods" +``` + +Example response: + +```json +[ + { + "id":1, + "freeze_start":"0 23 * * 5", + "freeze_end":"0 8 * * 1", + "cron_timezone":"UTC", + "created_at":"2020-05-15T17:03:35.702Z", + "updated_at":"2020-05-15T17:06:41.566Z" + } +] +``` + +## Get a Freeze Period by a `freeze_period_id` + +Get a Freeze Period for the given `freeze_period_id`. + +```plaintext +GET /projects/:id/freeze_periods/:freeze_period_id +``` + +| Attribute | Type | Required | Description | +| ------------- | -------------- | -------- | ----------------------------------------------------------------------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `freeze_period_id` | string | yes | The database ID of the Freeze Period. | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: gVWYVHDRzXiRpN1rUC8T" "https://gitlab.example.com/api/v4/projects/19/freeze_periods/1" +``` + +Example response: + +```json +{ + "id":1, + "freeze_start":"0 23 * * 5", + "freeze_end":"0 8 * * 1", + "cron_timezone":"UTC", + "created_at":"2020-05-15T17:03:35.702Z", + "updated_at":"2020-05-15T17:06:41.566Z" +} +``` + +## Create a Freeze Period + +Create a Freeze Period. + +```plaintext +POST /projects/:id/freeze_periods +``` + +| Attribute | Type | Required | Description | +| -------------------| --------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `freeze_start` | string | yes | Start of the Freeze Period in [cron](https://crontab.guru/) format. | +| `freeze_end` | string | yes | End of the Freeze Period in [cron](https://crontab.guru/) format. | +| `cron_timezone` | string | no | The timezone for the cron fields, defaults to UTC if not provided. | + +Example request: + +```shell +curl --header 'Content-Type: application/json' --header "PRIVATE-TOKEN: gVWYVHDRzXiRpN1rUC8T" \ + --data '{ "freeze_start": "0 23 * * 5", "freeze_end": "0 7 * * 1", "cron_timezone": "UTC" }' \ + --request POST https://gitlab.example.com/api/v4/projects/19/freeze_periods +``` + +Example response: + +```json +{ + "id":1, + "freeze_start":"0 23 * * 5", + "freeze_end":"0 7 * * 1", + "cron_timezone":"UTC", + "created_at":"2020-05-15T17:03:35.702Z", + "updated_at":"2020-05-15T17:03:35.702Z" +} +``` + +## Update a Freeze Period + +Update a Freeze Period for the given `freeze_period_id`. + +```plaintext +PUT /projects/:id/freeze_periods/:tag_name +``` + +| Attribute | Type | Required | Description | +| ------------- | --------------- | -------- | ----------------------------------------------------------------------------------------------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `freeze_period_id` | integer or string | yes | The database ID of the Freeze Period. | +| `freeze_start` | string | no | Start of the Freeze Period in [cron](https://crontab.guru/) format. | +| `freeze_end` | string | no | End of the Freeze Period in [cron](https://crontab.guru/) format. | +| `cron_timezone` | string | no | The timezone for the cron fields. | + +Example request: + +```shell +curl --header 'Content-Type: application/json' --header "PRIVATE-TOKEN: gVWYVHDRzXiRpN1rUC8T" \ + --data '{ "freeze_end": "0 8 * * 1" }' \ + --request PUT https://gitlab.example.com/api/v4/projects/19/freeze_periods/1 +``` + +Example response: + +```json +{ + "id":1, + "freeze_start":"0 23 * * 5", + "freeze_end":"0 8 * * 1", + "cron_timezone":"UTC", + "created_at":"2020-05-15T17:03:35.702Z", + "updated_at":"2020-05-15T17:06:41.566Z" +} +``` + +## Delete a Freeze Period + +Delete a Freeze Period for the given `freeze_period_id`. + +```plaintext +DELETE /projects/:id/freeze_periods/:freeze_period_id +``` + +| Attribute | Type | Required | Description | +| ------------- | -------------- | -------- | ----------------------------------------------------------------------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `freeze_period_id` | string | yes | The database ID of the Freeze Period. | + +Example request: + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: gVWYVHDRzXiRpN1rUC8T" "https://gitlab.example.com/api/v4/projects/19/freeze_periods/1" + +``` diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index baaa2e2f09f..820f78853d0 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -364,6 +364,9 @@ Example response: "last_successful_status_check_timestamp": 1510125024, "version": "10.3.0", "revision": "33d33a096a", + "package_files_count": 10, + "package_files_checksummed_count": 10, + "package_files_checksum_failed_count": 0 }, { "geo_node_id": 2, @@ -431,7 +434,10 @@ Example response: "cursor_last_event_timestamp": 1509681166, "last_successful_status_check_timestamp": 1510125024, "version": "10.3.0", - "revision": "33d33a096a" + "revision": "33d33a096a", + "package_files_count": 10, + "package_files_checksummed_count": 10, + "package_files_checksum_failed_count": 0 } ] ``` diff --git a/doc/api/graphql/getting_started.md b/doc/api/graphql/getting_started.md index 07147340ab4..901bf85ff40 100644 --- a/doc/api/graphql/getting_started.md +++ b/doc/api/graphql/getting_started.md @@ -2,7 +2,7 @@ This guide demonstrates basic usage of GitLab's GraphQL API. -See the [GraphQL API StyleGuide](../../development/api_graphql_styleguide.md) for implementation details +See the [GraphQL API style guide](../../development/api_graphql_styleguide.md) for implementation details aimed at developers who wish to work on developing the API itself. ## Running examples diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 9cdc83d584c..91f1413943c 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -33,12 +33,47 @@ type AddAwardEmojiPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! } """ +Autogenerated input type of AddProjectToSecurityDashboard +""" +input AddProjectToSecurityDashboardInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the project to be added to Instance Security Dashboard + """ + id: ID! +} + +""" +Autogenerated return type of AddProjectToSecurityDashboard +""" +type AddProjectToSecurityDashboardPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + Project that was added to the Instance Security Dashboard + """ + project: Project +} + +""" Autogenerated input type of AdminSidekiqQueuesDeleteJobs """ input AdminSidekiqQueuesDeleteJobsInput { @@ -93,7 +128,7 @@ type AdminSidekiqQueuesDeleteJobsPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -104,6 +139,311 @@ type AdminSidekiqQueuesDeleteJobsPayload { } """ +Describes an alert from the project's Alert Management +""" +type AlertManagementAlert { + """ + Timestamp the alert was created + """ + createdAt: Time + + """ + Description of the alert + """ + description: String + + """ + Alert details + """ + details: JSON + + """ + Timestamp the alert ended + """ + endedAt: Time + + """ + Number of events of this alert + """ + eventCount: Int + + """ + List of hosts the alert came from + """ + hosts: [String!] + + """ + Internal ID of the alert + """ + iid: ID! + + """ + Internal ID of the GitLab issue attached to the alert + """ + issueIid: ID + + """ + Monitoring tool the alert came from + """ + monitoringTool: String + + """ + Service the alert came from + """ + service: String + + """ + Severity of the alert + """ + severity: AlertManagementSeverity + + """ + Timestamp the alert was raised + """ + startedAt: Time + + """ + Status of the alert + """ + status: AlertManagementStatus + + """ + Title of the alert + """ + title: String + + """ + Timestamp the alert was last updated + """ + updatedAt: Time +} + +""" +The connection type for AlertManagementAlert. +""" +type AlertManagementAlertConnection { + """ + A list of edges. + """ + edges: [AlertManagementAlertEdge] + + """ + A list of nodes. + """ + nodes: [AlertManagementAlert] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type AlertManagementAlertEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: AlertManagementAlert +} + +""" +Values for sorting alerts +""" +enum AlertManagementAlertSort { + """ + Created time by ascending order + """ + CREATED_TIME_ASC + + """ + Created time by descending order + """ + CREATED_TIME_DESC + + """ + End time by ascending order + """ + END_TIME_ASC + + """ + End time by descending order + """ + END_TIME_DESC + + """ + Events count by ascending order + """ + EVENTS_COUNT_ASC + + """ + Events count by descending order + """ + EVENTS_COUNT_DESC + + """ + Severity by ascending order + """ + SEVERITY_ASC + + """ + Severity by descending order + """ + SEVERITY_DESC + + """ + Start time by ascending order + """ + START_TIME_ASC + + """ + Start time by descending order + """ + START_TIME_DESC + + """ + Status by ascending order + """ + STATUS_ASC + + """ + Status by descending order + """ + STATUS_DESC + + """ + Created time by ascending order + """ + UPDATED_TIME_ASC + + """ + Created time by descending order + """ + UPDATED_TIME_DESC + + """ + Created at ascending order + """ + created_asc + + """ + Created at descending order + """ + created_desc + + """ + Updated at ascending order + """ + updated_asc + + """ + Updated at descending order + """ + updated_desc +} + +""" +Represents total number of alerts for the represented categories +""" +type AlertManagementAlertStatusCountsType { + """ + Number of alerts with status ACKNOWLEDGED for the project + """ + acknowledged: Int + + """ + Total number of alerts for the project + """ + all: Int + + """ + Number of alerts with status IGNORED for the project + """ + ignored: Int + + """ + Number of alerts with status TRIGGERED or ACKNOWLEDGED for the project + """ + open: Int + + """ + Number of alerts with status RESOLVED for the project + """ + resolved: Int + + """ + Number of alerts with status TRIGGERED for the project + """ + triggered: Int +} + +""" +Alert severity values +""" +enum AlertManagementSeverity { + """ + Critical severity + """ + CRITICAL + + """ + High severity + """ + HIGH + + """ + Info severity + """ + INFO + + """ + Low severity + """ + LOW + + """ + Medium severity + """ + MEDIUM + + """ + Unknown severity + """ + UNKNOWN +} + +""" +Alert status values +""" +enum AlertManagementStatus { + """ + Acknowledged status + """ + ACKNOWLEDGED + + """ + Ignored status + """ + IGNORED + + """ + Resolved status + """ + RESOLVED + + """ + Triggered status + """ + TRIGGERED +} + +""" An emoji awarded by a user. """ type AwardEmoji { @@ -246,6 +586,31 @@ type Board { id: ID! """ + Lists of the project board + """ + lists( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): BoardListConnection + + """ Name of the board """ name: String @@ -291,6 +656,163 @@ type BoardEdge { node: Board } +""" +Represents a list for an issue board +""" +type BoardList { + """ + Assignee in the list + """ + assignee: User + + """ + Indicates if list is collapsed for this user + """ + collapsed: Boolean + + """ + ID (global ID) of the list + """ + id: ID! + + """ + Label of the list + """ + label: Label + + """ + The current limit metric for the list + """ + limitMetric: ListLimitMetric + + """ + Type of the list + """ + listType: String! + + """ + Maximum number of issues in the list + """ + maxIssueCount: Int + + """ + Maximum weight of issues in the list + """ + maxIssueWeight: Int + + """ + Milestone of the list + """ + milestone: Milestone + + """ + Position of list within the board + """ + position: Int + + """ + Title of the list + """ + title: String! +} + +""" +The connection type for BoardList. +""" +type BoardListConnection { + """ + A list of edges. + """ + edges: [BoardListEdge] + + """ + A list of nodes. + """ + nodes: [BoardList] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type BoardListEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: BoardList +} + +""" +Autogenerated input type of BoardListUpdateLimitMetrics +""" +input BoardListUpdateLimitMetricsInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The new limit metric type for the list. + """ + limitMetric: ListLimitMetric + + """ + The global ID of the list. + """ + listId: ID! + + """ + The new maximum issue count limit. + """ + maxIssueCount: Int + + """ + The new maximum issue weight limit. + """ + maxIssueWeight: Int +} + +""" +Autogenerated return type of BoardListUpdateLimitMetrics +""" +type BoardListUpdateLimitMetricsPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The updated list + """ + list: BoardList +} + +type Branch { + """ + Commit for the branch + """ + commit: Commit + + """ + Name of the branch + """ + name: String! +} + type Commit { """ Author of the commit @@ -403,12 +925,167 @@ type Commit { title: String """ + The GitLab Flavored Markdown rendering of `title` + """ + titleHtml: String + + """ Web URL of the commit """ webUrl: String! } """ +Autogenerated input type of CreateAlertIssue +""" +input CreateAlertIssueInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The iid of the alert to mutate + """ + iid: String! + + """ + The project the alert to mutate is in + """ + projectPath: ID! +} + +""" +Autogenerated return type of CreateAlertIssue +""" +type CreateAlertIssuePayload { + """ + The alert after mutation + """ + alert: AlertManagementAlert + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue created after mutation + """ + issue: Issue +} + +""" +Autogenerated input type of CreateAnnotation +""" +input CreateAnnotationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The global id of the cluster to add an annotation to + """ + clusterId: ID + + """ + The path to a file defining the dashboard on which the annotation should be added + """ + dashboardPath: String! + + """ + The description of the annotation + """ + description: String! + + """ + Timestamp indicating ending moment to which the annotation relates + """ + endingAt: Time + + """ + The global id of the environment to add an annotation to + """ + environmentId: ID + + """ + Timestamp indicating starting moment to which the annotation relates + """ + startingAt: Time! +} + +""" +Autogenerated return type of CreateAnnotation +""" +type CreateAnnotationPayload { + """ + The created annotation + """ + annotation: MetricsDashboardAnnotation + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" +Autogenerated input type of CreateBranch +""" +input CreateBranchInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Name of the branch + """ + name: String! + + """ + Project full path the branch is associated with + """ + projectPath: ID! + + """ + Branch name or commit SHA to create branch from + """ + ref: String! +} + +""" +Autogenerated return type of CreateBranch +""" +type CreateBranchPayload { + """ + Branch after mutation + """ + branch: Branch + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of CreateDiffNote """ input CreateDiffNoteInput { @@ -443,7 +1120,7 @@ type CreateDiffNotePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -528,7 +1205,7 @@ type CreateEpicPayload { epic: Epic """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! } @@ -568,7 +1245,7 @@ type CreateImageDiffNotePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -579,6 +1256,61 @@ type CreateImageDiffNotePayload { } """ +Autogenerated input type of CreateIteration +""" +input CreateIterationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The description of the iteration + """ + description: String + + """ + The end date of the iteration + """ + dueDate: String + + """ + The target group for the iteration + """ + groupPath: ID! + + """ + The start date of the iteration + """ + startDate: String + + """ + The title of the iteration + """ + title: String +} + +""" +Autogenerated return type of CreateIteration +""" +type CreateIterationPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The created iteration + """ + iteration: Iteration +} + +""" Autogenerated input type of CreateNote """ input CreateNoteInput { @@ -613,7 +1345,7 @@ type CreateNotePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -653,7 +1385,7 @@ type CreateRequirementPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -698,6 +1430,11 @@ input CreateSnippetInput { title: String! """ + The paths to files uploaded in the snippet description + """ + uploadedFiles: [String!] + + """ The visibility level of the snippet """ visibilityLevel: VisibilityLevelsEnum! @@ -713,7 +1450,7 @@ type CreateSnippetPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -1258,7 +1995,7 @@ type DesignManagementDeletePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -1308,7 +2045,7 @@ type DesignManagementUploadPayload { designs: [Design!]! """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -1498,7 +2235,7 @@ type DestroyNotePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -1533,7 +2270,7 @@ type DestroySnippetPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -1862,7 +2599,7 @@ type DismissVulnerabilityPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -2415,7 +3152,7 @@ type EpicAddIssuePayload { epicIssue: EpicIssue """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! } @@ -2650,6 +3387,11 @@ type EpicIssue implements Noteable { iid: ID! """ + Iteration of the issue + """ + iteration: Iteration + + """ Labels of the issue """ labels( @@ -2940,7 +3682,7 @@ type EpicSetSubscriptionPayload { epic: Epic """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! } @@ -3001,7 +3743,7 @@ input EpicTreeNodeFieldsInputType { """ The id of the epic_issue or issue that the actual epic or issue is switched with """ - adjacentReferenceId: ID! + adjacentReferenceId: ID """ The id of the epic_issue or epic that is being moved @@ -3016,7 +3758,7 @@ input EpicTreeNodeFieldsInputType { """ The type of the switch, after or before allowed """ - relativePosition: MoveType! + relativePosition: MoveType } """ @@ -3049,7 +3791,7 @@ type EpicTreeReorderPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! } @@ -3091,6 +3833,36 @@ type GeoNode { name: String """ + Package file registries of the GeoNode. Available only when feature flag `geo_self_service_framework` is enabled + """ + packageFileRegistries( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Filters registries by their ID + """ + ids: [ID!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): PackageFileRegistryConnection + + """ Indicates whether this Geo node is the primary """ primary: Boolean @@ -3163,7 +3935,7 @@ type GrafanaIntegration { enabled: Boolean! """ - Url for the Grafana host for the Grafana integration + URL for the Grafana host for the Grafana integration """ grafanaUrl: String! @@ -3509,6 +4281,53 @@ type Group { ): IssueConnection """ + Find iterations + """ + iterations( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + List items within a time frame where items.end_date is between startDate and + endDate parameters (startDate parameter must be present) + """ + endDate: Time + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + List items within a time frame where items.start_date is between startDate + and endDate parameters (endDate parameter must be present) + """ + startDate: Time + + """ + Filter iterations by state + """ + state: IterationState + + """ + Fuzzy search by title + """ + title: String + ): IterationConnection + + """ Indicates if Large File Storage (LFS) is enabled for namespace """ lfsEnabled: Boolean @@ -3544,6 +4363,11 @@ type Group { first: Int """ + Return also milestones in all subgroups and subprojects + """ + includeDescendants: Boolean + + """ Returns the last _n_ elements from the list. """ last: Int @@ -3600,6 +4424,11 @@ type Group { first: Int """ + Returns only the projects which have vulnerabilities + """ + hasVulnerabilities: Boolean = false + + """ Include also subgroup projects """ includeSubgroups: Boolean = false @@ -3650,9 +4479,14 @@ type Group { before: String """ - List time logs within a time range where the logged date is before end_date parameter. + List time logs within a date range where the logged date is equal to or before endDate """ - endDate: Time! + endDate: Time + + """ + List time-logs within a time range where the logged time is equal to or before endTime + """ + endTime: Time """ Returns the first _n_ elements from the list. @@ -3665,9 +4499,14 @@ type Group { last: Int """ - List time logs within a time range where the logged date is after start_date parameter. + List time logs within a date range where the logged date is equal to or after startDate """ - startDate: Time! + startDate: Time + + """ + List time-logs within a time range where the logged time is equal to or after startTime + """ + startTime: Time ): TimelogConnection! """ @@ -3686,8 +4525,7 @@ type Group { visibility: String """ - Vulnerabilities reported on the projects in the group and its subgroups. - Available only when feature flag `first_class_vulnerabilities` is enabled + Vulnerabilities reported on the projects in the group and its subgroups """ vulnerabilities( """ @@ -3732,6 +4570,41 @@ type Group { ): VulnerabilityConnection """ + Number of vulnerabilities per severity level, per day, for the projects in the group and its subgroups + """ + vulnerabilitiesCountByDayAndSeverity( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Last day for which to fetch vulnerability history + """ + endDate: ISO8601Date! + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + First day for which to fetch vulnerability history + """ + startDate: ISO8601Date! + ): VulnerabilitiesCountByDayAndSeverityConnection + + """ Web URL of the group """ webUrl: String! @@ -3754,6 +4627,38 @@ enum HealthStatus { } """ +An ISO 8601-encoded date +""" +scalar ISO8601Date + +type InstanceSecurityDashboard { + """ + Projects selected in Instance Security Dashboard + """ + projects( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ProjectConnection! +} + +""" State of a GitLab issue or merge request """ enum IssuableState { @@ -3884,6 +4789,11 @@ type Issue implements Noteable { iid: ID! """ + Iteration of the issue + """ + iteration: Iteration + + """ Labels of the issue """ labels( @@ -4164,7 +5074,7 @@ type IssueSetConfidentialPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -4209,7 +5119,52 @@ type IssueSetDueDatePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" +Autogenerated input type of IssueSetIteration +""" +input IssueSetIterationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The iid of the issue to mutate + """ + iid: String! + + """ + The iteration to assign to the issue. + """ + iterationId: ID + + """ + The project the issue to mutate is in + """ + projectPath: ID! +} + +""" +Autogenerated return type of IssueSetIteration +""" +type IssueSetIterationPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -4254,7 +5209,7 @@ type IssueSetWeightPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -4279,6 +5234,36 @@ enum IssueSort { DUE_DATE_DESC """ + Label priority by ascending order + """ + LABEL_PRIORITY_ASC + + """ + Label priority by descending order + """ + LABEL_PRIORITY_DESC + + """ + Milestone due date by ascending order + """ + MILESTONE_DUE_ASC + + """ + Milestone due date by descending order + """ + MILESTONE_DUE_DESC + + """ + Priority by ascending order + """ + PRIORITY_ASC + + """ + Priority by descending order + """ + PRIORITY_DESC + + """ Relative position by ascending order """ RELATIVE_POSITION_ASC @@ -4324,18 +5309,124 @@ enum IssueState { } """ +Represents an iteration object. +""" +type Iteration { + """ + Timestamp of iteration creation + """ + createdAt: Time! + + """ + Description of the iteration + """ + description: String + + """ + Timestamp of the iteration due date + """ + dueDate: Time + + """ + ID of the iteration + """ + id: ID! + + """ + Timestamp of the iteration start date + """ + startDate: Time + + """ + State of the iteration + """ + state: IterationState! + + """ + Title of the iteration + """ + title: String! + + """ + Timestamp of last iteration update + """ + updatedAt: Time! + + """ + Web path of the iteration + """ + webPath: String! + + """ + Web URL of the iteration + """ + webUrl: String! +} + +""" +The connection type for Iteration. +""" +type IterationConnection { + """ + A list of edges. + """ + edges: [IterationEdge] + + """ + A list of nodes. + """ + nodes: [Iteration] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type IterationEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: Iteration +} + +""" +State of a GitLab iteration +""" +enum IterationState { + all + closed + opened + started + upcoming +} + +""" Represents untyped JSON """ scalar JSON type JiraImport { """ + Timestamp of when the Jira import was created + """ + createdAt: Time + + """ Project key for the imported Jira project """ jiraProjectKey: String! """ - Timestamp of when the Jira import was created + Timestamp of when the Jira import was scheduled """ scheduledAt: Time @@ -4415,7 +5506,7 @@ type JiraImportStartPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -4505,6 +5596,15 @@ type LabelEdge { } """ +List limit metric setting +""" +enum ListLimitMetric { + all_metrics + issue_count + issue_weights +} + +""" Autogenerated input type of MarkAsSpamSnippet """ input MarkAsSpamSnippetInput { @@ -4529,7 +5629,7 @@ type MarkAsSpamSnippetPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -5076,7 +6176,7 @@ type MergeRequestSetAssigneesPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -5126,7 +6226,7 @@ type MergeRequestSetLabelsPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -5171,7 +6271,7 @@ type MergeRequestSetLockedPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -5216,7 +6316,7 @@ type MergeRequestSetMilestonePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -5261,7 +6361,7 @@ type MergeRequestSetSubscriptionPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -5306,7 +6406,7 @@ type MergeRequestSetWipPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -5340,7 +6440,7 @@ type Metadata { type MetricsDashboard { """ - Annotations added to the dashboard. Will always return `null` if `metrics_dashboard_annotations` feature flag is disabled + Annotations added to the dashboard """ annotations( """ @@ -5389,7 +6489,7 @@ type MetricsDashboardAnnotation { """ Timestamp marking end of annotated time span """ - endingAt: String + endingAt: Time """ ID of the annotation @@ -5404,7 +6504,7 @@ type MetricsDashboardAnnotation { """ Timestamp marking start of annotated time span """ - startingAt: String + startingAt: Time } """ @@ -5549,10 +6649,16 @@ enum MoveType { type Mutation { addAwardEmoji(input: AddAwardEmojiInput!): AddAwardEmojiPayload + addProjectToSecurityDashboard(input: AddProjectToSecurityDashboardInput!): AddProjectToSecurityDashboardPayload adminSidekiqQueuesDeleteJobs(input: AdminSidekiqQueuesDeleteJobsInput!): AdminSidekiqQueuesDeleteJobsPayload + boardListUpdateLimitMetrics(input: BoardListUpdateLimitMetricsInput!): BoardListUpdateLimitMetricsPayload + createAlertIssue(input: CreateAlertIssueInput!): CreateAlertIssuePayload + createAnnotation(input: CreateAnnotationInput!): CreateAnnotationPayload + createBranch(input: CreateBranchInput!): CreateBranchPayload createDiffNote(input: CreateDiffNoteInput!): CreateDiffNotePayload createEpic(input: CreateEpicInput!): CreateEpicPayload createImageDiffNote(input: CreateImageDiffNoteInput!): CreateImageDiffNotePayload + createIteration(input: CreateIterationInput!): CreateIterationPayload createNote(input: CreateNoteInput!): CreateNotePayload createRequirement(input: CreateRequirementInput!): CreateRequirementPayload createSnippet(input: CreateSnippetInput!): CreateSnippetPayload @@ -5566,6 +6672,7 @@ type Mutation { epicTreeReorder(input: EpicTreeReorderInput!): EpicTreeReorderPayload issueSetConfidential(input: IssueSetConfidentialInput!): IssueSetConfidentialPayload issueSetDueDate(input: IssueSetDueDateInput!): IssueSetDueDatePayload + issueSetIteration(input: IssueSetIterationInput!): IssueSetIterationPayload issueSetWeight(input: IssueSetWeightInput!): IssueSetWeightPayload jiraImportStart(input: JiraImportStartInput!): JiraImportStartPayload markAsSpamSnippet(input: MarkAsSpamSnippetInput!): MarkAsSpamSnippetPayload @@ -5576,11 +6683,13 @@ type Mutation { mergeRequestSetSubscription(input: MergeRequestSetSubscriptionInput!): MergeRequestSetSubscriptionPayload mergeRequestSetWip(input: MergeRequestSetWipInput!): MergeRequestSetWipPayload removeAwardEmoji(input: RemoveAwardEmojiInput!): RemoveAwardEmojiPayload + removeProjectFromSecurityDashboard(input: RemoveProjectFromSecurityDashboardInput!): RemoveProjectFromSecurityDashboardPayload todoMarkDone(input: TodoMarkDoneInput!): TodoMarkDonePayload todoRestore(input: TodoRestoreInput!): TodoRestorePayload todoRestoreMany(input: TodoRestoreManyInput!): TodoRestoreManyPayload todosMarkAllDone(input: TodosMarkAllDoneInput!): TodosMarkAllDonePayload toggleAwardEmoji(input: ToggleAwardEmojiInput!): ToggleAwardEmojiPayload + updateAlertStatus(input: UpdateAlertStatusInput!): UpdateAlertStatusPayload updateEpic(input: UpdateEpicInput!): UpdateEpicPayload """ @@ -5681,6 +6790,11 @@ type Namespace { first: Int """ + Returns only the projects which have vulnerabilities + """ + hasVulnerabilities: Boolean = false + + """ Include also subgroup projects """ includeSubgroups: Boolean = false @@ -5934,6 +7048,188 @@ interface Noteable { } """ +Represents a package +""" +type Package { + """ + The created date + """ + createdAt: Time! + + """ + The ID of the package + """ + id: ID! + + """ + The name of the package + """ + name: String! + + """ + The type of the package + """ + packageType: PackageTypeEnum! + + """ + The update date + """ + updatedAt: Time! + + """ + The version of the package + """ + version: String +} + +""" +The connection type for Package. +""" +type PackageConnection { + """ + A list of edges. + """ + edges: [PackageEdge] + + """ + A list of nodes. + """ + nodes: [Package] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type PackageEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: Package +} + +""" +Represents the sync and verification state of a package file +""" +type PackageFileRegistry { + """ + Timestamp when the PackageFileRegistry was created + """ + createdAt: Time + + """ + ID of the PackageFileRegistry + """ + id: ID! + + """ + Error message during sync of the PackageFileRegistry + """ + lastSyncFailure: String + + """ + Timestamp of the most recent successful sync of the PackageFileRegistry + """ + lastSyncedAt: Time + + """ + ID of the PackageFile + """ + packageFileId: ID! + + """ + Timestamp after which the PackageFileRegistry should be resynced + """ + retryAt: Time + + """ + Number of consecutive failed sync attempts of the PackageFileRegistry + """ + retryCount: Int + + """ + Sync state of the PackageFileRegistry + """ + state: RegistryState +} + +""" +The connection type for PackageFileRegistry. +""" +type PackageFileRegistryConnection { + """ + A list of edges. + """ + edges: [PackageFileRegistryEdge] + + """ + A list of nodes. + """ + nodes: [PackageFileRegistry] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type PackageFileRegistryEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: PackageFileRegistry +} + +enum PackageTypeEnum { + """ + Packages from the composer package manager + """ + COMPOSER + + """ + Packages from the conan package manager + """ + CONAN + + """ + Packages from the maven package manager + """ + MAVEN + + """ + Packages from the npm package manager + """ + NPM + + """ + Packages from the nuget package manager + """ + NUGET + + """ + Packages from the pypi package manager + """ + PYPI +} + +""" Information about pagination in a connection. """ type PageInfo { @@ -6099,6 +7395,81 @@ enum PipelineStatusEnum { type Project { """ + A single Alert Management alert of the project + """ + alertManagementAlert( + """ + IID of the alert. For example, "1" + """ + iid: String + + """ + Search criteria for filtering alerts. This will search on title, description, service, monitoring_tool. + """ + search: String + + """ + Sort alerts by this criteria + """ + sort: AlertManagementAlertSort + + """ + Alerts with the specified statues. For example, [TRIGGERED] + """ + statuses: [AlertManagementStatus!] + ): AlertManagementAlert + + """ + Counts of alerts by status for the project + """ + alertManagementAlertStatusCounts: AlertManagementAlertStatusCountsType + + """ + Alert Management alerts of the project + """ + alertManagementAlerts( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + IID of the alert. For example, "1" + """ + iid: String + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Search criteria for filtering alerts. This will search on title, description, service, monitoring_tool. + """ + search: String + + """ + Sort alerts by this criteria + """ + sort: AlertManagementAlertSort + + """ + Alerts with the specified statues. For example, [TRIGGERED] + """ + statuses: [AlertManagementStatus!] + ): AlertManagementAlertConnection + + """ Indicates the archived status of the project """ archived: Boolean @@ -6571,6 +7942,31 @@ type Project { openIssuesCount: Int """ + Packages of the project + """ + packages( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): PackageConnection + + """ Path of the project """ path: String! @@ -6627,6 +8023,41 @@ type Project { publicJobs: Boolean """ + A single release of the project. Available only when feature flag `graphql_release_data` is enabled + """ + release( + """ + The name of the tag associated to the release + """ + tagName: String! + ): Release + + """ + Releases of the project. Available only when feature flag `graphql_release_data` is enabled + """ + releases( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ReleaseConnection + + """ Indicates if `Delete source branch` option should be enabled by default for all new merge requests of the project """ removeSourceBranchAfterMerge: Boolean @@ -6857,7 +8288,7 @@ type Project { visibility: String """ - Vulnerabilities reported on the project. Available only when feature flag `first_class_vulnerabilities` is enabled + Vulnerabilities reported on the project """ vulnerabilities( """ @@ -6902,8 +8333,7 @@ type Project { ): VulnerabilityConnection """ - Counts for each severity of vulnerability of the project. Available only when - feature flag `first_class_vulnerabilities` is enabled + Counts for each severity of vulnerability of the project """ vulnerabilitySeveritiesCount: VulnerabilitySeveritiesCount @@ -7244,6 +8674,11 @@ type Query { ): Group """ + Fields related to Instance Security Dashboard + """ + instanceSecurityDashboard: InstanceSecurityDashboard + + """ Metadata about GitLab """ metadata: Metadata @@ -7269,6 +8704,41 @@ type Query { ): Project """ + Find projects visible to the current user + """ + projects( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Limit projects that the current user is a member of + """ + membership: Boolean + + """ + Search criteria + """ + search: String + ): ProjectConnection + + """ Find Snippets visible to the current user """ snippets( @@ -7367,6 +8837,173 @@ type Query { """ state: [VulnerabilityState!] ): VulnerabilityConnection + + """ + Number of vulnerabilities per severity level, per day, for the projects on the current user's instance security dashboard + """ + vulnerabilitiesCountByDayAndSeverity( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Last day for which to fetch vulnerability history + """ + endDate: ISO8601Date! + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + First day for which to fetch vulnerability history + """ + startDate: ISO8601Date! + ): VulnerabilitiesCountByDayAndSeverityConnection +} + +""" +State of a Geo registry. +""" +enum RegistryState { + """ + Registry that failed to sync + """ + FAILED + + """ + Registry waiting to be synced + """ + PENDING + + """ + Registry currently syncing + """ + STARTED + + """ + Registry that is synced + """ + SYNCED +} + +type Release { + """ + User that created the release + """ + author: User + + """ + The commit associated with the release + """ + commit: Commit + + """ + Timestamp of when the release was created + """ + createdAt: Time + + """ + Description (also known as "release notes") of the release + """ + description: String + + """ + The GitLab Flavored Markdown rendering of `description` + """ + descriptionHtml: String + + """ + Milestones associated to the release + """ + milestones( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): MilestoneConnection + + """ + Name of the release + """ + name: String + + """ + Timestamp of when the release was released + """ + releasedAt: Time + + """ + Name of the tag associated with the release + """ + tagName: String! + + """ + Relative web path to the tag associated with the release + """ + tagPath: String +} + +""" +The connection type for Release. +""" +type ReleaseConnection { + """ + A list of edges. + """ + edges: [ReleaseEdge] + + """ + A list of nodes. + """ + nodes: [Release] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ReleaseEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: Release } """ @@ -7404,7 +9041,37 @@ type RemoveAwardEmojiPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" +Autogenerated input type of RemoveProjectFromSecurityDashboard +""" +input RemoveProjectFromSecurityDashboardInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + ID of the project to remove from the Instance Security Dashboard + """ + id: ID! +} + +""" +Autogenerated return type of RemoveProjectFromSecurityDashboard +""" +type RemoveProjectFromSecurityDashboardPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. """ errors: [String!]! } @@ -8123,7 +9790,6 @@ enum ServiceType { HANGOUTS_CHAT_SERVICE HIPCHAT_SERVICE IRKER_SERVICE - JENKINS_DEPRECATED_SERVICE JENKINS_SERVICE JIRA_SERVICE MATTERMOST_SERVICE @@ -8139,6 +9805,7 @@ enum ServiceType { SLACK_SLASH_COMMANDS_SERVICE TEAMCITY_SERVICE UNIFY_CIRCUIT_SERVICE + WEBEX_TEAMS_SERVICE YOUTRACK_SERVICE } @@ -8207,7 +9874,7 @@ type Snippet implements Noteable { httpUrlToRepo: String """ - Id of the snippet + ID of the snippet """ id: ID! @@ -8287,6 +9954,11 @@ type SnippetBlob { binary: Boolean! """ + Blob external storage + """ + externalStorage: String + + """ Blob mode """ mode: String @@ -8312,6 +9984,11 @@ type SnippetBlob { rawPath: String! """ + Shows whether the blob is rendered as text + """ + renderedAsText: Boolean! + + """ Blob highlighted data """ richData: String @@ -8653,7 +10330,7 @@ type Todo { group: Group """ - Id of the todo + ID of the todo """ id: ID! @@ -8743,7 +10420,7 @@ type TodoMarkDonePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -8793,7 +10470,7 @@ type TodoRestoreManyPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -8813,7 +10490,7 @@ type TodoRestorePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -8875,7 +10552,7 @@ type TodosMarkAllDonePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -8920,7 +10597,7 @@ type ToggleAwardEmojiPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -9092,6 +10769,56 @@ enum TypeEnum { project } +""" +Autogenerated input type of UpdateAlertStatus +""" +input UpdateAlertStatusInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The iid of the alert to mutate + """ + iid: String! + + """ + The project the alert to mutate is in + """ + projectPath: ID! + + """ + The status to set the alert + """ + status: AlertManagementStatus! +} + +""" +Autogenerated return type of UpdateAlertStatus +""" +type UpdateAlertStatusPayload { + """ + The alert after mutation + """ + alert: AlertManagementAlert + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue created after mutation + """ + issue: Issue +} + input UpdateDiffImagePositionInput { """ Total height of the image @@ -9199,7 +10926,7 @@ type UpdateEpicPayload { epic: Epic """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! } @@ -9239,7 +10966,7 @@ type UpdateImageDiffNotePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -9304,7 +11031,7 @@ type UpdateIssuePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -9344,7 +11071,7 @@ type UpdateNotePayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -9394,7 +11121,7 @@ type UpdateRequirementPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -9454,7 +11181,7 @@ type UpdateSnippetPayload { clientMutationId: String """ - Reasons why the mutation failed. + Errors encountered during execution of the mutation. """ errors: [String!]! @@ -9473,6 +11200,11 @@ type User { avatarUrl: String """ + ID of the user + """ + id: ID! + + """ Human-readable name of the user """ name: String! @@ -9518,6 +11250,11 @@ type User { ): SnippetConnection """ + State of the issue + """ + state: String! + + """ Todos of the user """ todos( @@ -9643,6 +11380,61 @@ enum VisibilityScopesEnum { } """ +Represents the number of vulnerabilities for a particular severity on a particular day +""" +type VulnerabilitiesCountByDayAndSeverity { + """ + Number of vulnerabilities + """ + count: Int + + """ + Date for the count + """ + day: ISO8601Date + + """ + Severity of the counted vulnerabilities + """ + severity: VulnerabilitySeverity +} + +""" +The connection type for VulnerabilitiesCountByDayAndSeverity. +""" +type VulnerabilitiesCountByDayAndSeverityConnection { + """ + A list of edges. + """ + edges: [VulnerabilitiesCountByDayAndSeverityEdge] + + """ + A list of nodes. + """ + nodes: [VulnerabilitiesCountByDayAndSeverity] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type VulnerabilitiesCountByDayAndSeverityEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: VulnerabilitiesCountByDayAndSeverity +} + +""" Represents a vulnerability. """ type Vulnerability { @@ -9657,10 +11449,9 @@ type Vulnerability { id: ID! """ - The JSON location metadata for the vulnerability. Its format depends on the - type of the security scan that found the vulnerability + Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability """ - location: JSON + location: VulnerabilityLocation """ The project on which the vulnerability was found @@ -9734,6 +11525,101 @@ type VulnerabilityEdge { } """ +Represents a vulnerability location. The fields with data will depend on the vulnerability report type +""" +union VulnerabilityLocation = VulnerabilityLocationContainerScanning | VulnerabilityLocationDast | VulnerabilityLocationDependencyScanning | VulnerabilityLocationSast + +""" +Represents the location of a vulnerability found by a container security scan +""" +type VulnerabilityLocationContainerScanning { + """ + Dependency containing the vulnerability + """ + dependency: VulnerableDependency + + """ + Name of the vulnerable container image + """ + image: String + + """ + Operating system that runs on the vulnerable container image + """ + operatingSystem: String +} + +""" +Represents the location of a vulnerability found by a DAST scan +""" +type VulnerabilityLocationDast { + """ + Domain name of the vulnerable request + """ + hostname: String + + """ + Query parameter for the URL on which the vulnerability occurred + """ + param: String + + """ + URL path and query string of the vulnerable request + """ + path: String + + """ + HTTP method of the vulnerable request + """ + requestMethod: String +} + +""" +Represents the location of a vulnerability found by a dependency security scan +""" +type VulnerabilityLocationDependencyScanning { + """ + Dependency containing the vulnerability + """ + dependency: VulnerableDependency + + """ + Path to the vulnerable file + """ + file: String +} + +""" +Represents the location of a vulnerability found by a SAST scan +""" +type VulnerabilityLocationSast { + """ + Number of the last relevant line in the vulnerable file + """ + endLine: String + + """ + Path to the vulnerable file + """ + file: String + + """ + Number of the first relevant line in the vulnerable file + """ + startLine: String + + """ + Class containing the vulnerability + """ + vulnerableClass: String + + """ + Method containing the vulnerability + """ + vulnerableMethod: String +} + +""" Check permissions for the current user on a vulnerability """ type VulnerabilityPermissions { @@ -9843,4 +11729,29 @@ enum VulnerabilityState { DETECTED DISMISSED RESOLVED +} + +""" +Represents a vulnerable dependency. Used in vulnerability location data +""" +type VulnerableDependency { + """ + The package associated with the vulnerable dependency + """ + package: VulnerablePackage + + """ + The version of the vulnerable dependency + """ + version: String +} + +""" +Represents a vulnerable package. Used in vulnerability dependency data +""" +type VulnerablePackage { + """ + The name of the vulnerable package + """ + name: String }
\ No newline at end of file diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 26a2acdb4c6..40bfa08cff3 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -93,7 +93,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -127,6 +127,108 @@ }, { "kind": "INPUT_OBJECT", + "name": "AddProjectToSecurityDashboardInput", + "description": "Autogenerated input type of AddProjectToSecurityDashboard", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "ID of the project to be added to Instance Security Dashboard", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AddProjectToSecurityDashboardPayload", + "description": "Autogenerated return type of AddProjectToSecurityDashboard", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "project", + "description": "Project that was added to the Instance Security Dashboard", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Project", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "AdminSidekiqQueuesDeleteJobsInput", "description": "Autogenerated input type of AdminSidekiqQueuesDeleteJobs", "fields": null, @@ -241,7 +343,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -289,6 +391,651 @@ }, { "kind": "OBJECT", + "name": "AlertManagementAlert", + "description": "Describes an alert from the project's Alert Management", + "fields": [ + { + "name": "createdAt", + "description": "Timestamp the alert was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "description", + "description": "Description of the alert", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "details", + "description": "Alert details", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JSON", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "endedAt", + "description": "Timestamp the alert ended", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "eventCount", + "description": "Number of events of this alert", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "hosts", + "description": "List of hosts the alert came from", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "iid", + "description": "Internal ID of the alert", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issueIid", + "description": "Internal ID of the GitLab issue attached to the alert", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "monitoringTool", + "description": "Monitoring tool the alert came from", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "service", + "description": "Service the alert came from", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "severity", + "description": "Severity of the alert", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "AlertManagementSeverity", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startedAt", + "description": "Timestamp the alert was raised", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "status", + "description": "Status of the alert", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "AlertManagementStatus", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title", + "description": "Title of the alert", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Timestamp the alert was last updated", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AlertManagementAlertConnection", + "description": "The connection type for AlertManagementAlert.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "AlertManagementAlertEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AlertManagementAlertEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "AlertManagementAlertSort", + "description": "Values for sorting alerts", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "updated_desc", + "description": "Updated at descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updated_asc", + "description": "Updated at ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_desc", + "description": "Created at descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_asc", + "description": "Created at ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "START_TIME_ASC", + "description": "Start time by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "START_TIME_DESC", + "description": "Start time by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "END_TIME_ASC", + "description": "End time by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "END_TIME_DESC", + "description": "End time by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CREATED_TIME_ASC", + "description": "Created time by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CREATED_TIME_DESC", + "description": "Created time by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "UPDATED_TIME_ASC", + "description": "Created time by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "UPDATED_TIME_DESC", + "description": "Created time by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "EVENTS_COUNT_ASC", + "description": "Events count by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "EVENTS_COUNT_DESC", + "description": "Events count by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "SEVERITY_ASC", + "description": "Severity by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "SEVERITY_DESC", + "description": "Severity by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "STATUS_ASC", + "description": "Status by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "STATUS_DESC", + "description": "Status by descending order", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AlertManagementAlertStatusCountsType", + "description": "Represents total number of alerts for the represented categories", + "fields": [ + { + "name": "acknowledged", + "description": "Number of alerts with status ACKNOWLEDGED for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "all", + "description": "Total number of alerts for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "ignored", + "description": "Number of alerts with status IGNORED for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "open", + "description": "Number of alerts with status TRIGGERED or ACKNOWLEDGED for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolved", + "description": "Number of alerts with status RESOLVED for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "triggered", + "description": "Number of alerts with status TRIGGERED for the project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "AlertManagementSeverity", + "description": "Alert severity values", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "CRITICAL", + "description": "Critical severity", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "HIGH", + "description": "High severity", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "MEDIUM", + "description": "Medium severity", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "LOW", + "description": "Low severity", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "INFO", + "description": "Info severity", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "UNKNOWN", + "description": "Unknown severity", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "AlertManagementStatus", + "description": "Alert status values", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "TRIGGERED", + "description": "Triggered status", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "ACKNOWLEDGED", + "description": "Acknowledged status", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "RESOLVED", + "description": "Resolved status", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "IGNORED", + "description": "Ignored status", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "AwardEmoji", "description": "An emoji awarded by a user.", "fields": [ @@ -771,6 +1518,59 @@ "deprecationReason": null }, { + "name": "lists", + "description": "Lists of the project board", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "BoardListConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "name", "description": "Name of the board", "args": [ @@ -919,6 +1719,429 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "BoardList", + "description": "Represents a list for an issue board", + "fields": [ + { + "name": "assignee", + "description": "Assignee in the list", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "collapsed", + "description": "Indicates if list is collapsed for this user", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID (global ID) of the list", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "label", + "description": "Label of the list", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Label", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "limitMetric", + "description": "The current limit metric for the list", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "ListLimitMetric", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "listType", + "description": "Type of the list", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "maxIssueCount", + "description": "Maximum number of issues in the list", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "maxIssueWeight", + "description": "Maximum weight of issues in the list", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "milestone", + "description": "Milestone of the list", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Milestone", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "position", + "description": "Position of list within the board", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title", + "description": "Title of the list", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "BoardListConnection", + "description": "The connection type for BoardList.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "BoardListEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "BoardList", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "BoardListEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardList", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "BoardListUpdateLimitMetricsInput", + "description": "Autogenerated input type of BoardListUpdateLimitMetrics", + "fields": null, + "inputFields": [ + { + "name": "listId", + "description": "The global ID of the list.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "limitMetric", + "description": "The new limit metric type for the list.", + "type": { + "kind": "ENUM", + "name": "ListLimitMetric", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "maxIssueCount", + "description": "The new maximum issue count limit.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "maxIssueWeight", + "description": "The new maximum issue weight limit.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "BoardListUpdateLimitMetricsPayload", + "description": "Autogenerated return type of BoardListUpdateLimitMetrics", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "list", + "description": "The updated list", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "BoardList", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "SCALAR", "name": "Boolean", "description": "Represents `true` or `false` values.", @@ -930,6 +2153,51 @@ }, { "kind": "OBJECT", + "name": "Branch", + "description": null, + "fields": [ + { + "name": "commit", + "description": "Commit for the branch", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Commit", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the branch", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Commit", "description": null, "fields": [ @@ -1208,6 +2476,20 @@ "deprecationReason": null }, { + "name": "titleHtml", + "description": "The GitLab Flavored Markdown rendering of `title`", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL of the commit", "args": [ @@ -1235,6 +2517,426 @@ }, { "kind": "INPUT_OBJECT", + "name": "CreateAlertIssueInput", + "description": "Autogenerated input type of CreateAlertIssue", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the alert to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the alert to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateAlertIssuePayload", + "description": "Autogenerated return type of CreateAlertIssue", + "fields": [ + { + "name": "alert", + "description": "The alert after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue created after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "CreateAnnotationInput", + "description": "Autogenerated input type of CreateAnnotation", + "fields": null, + "inputFields": [ + { + "name": "environmentId", + "description": "The global id of the environment to add an annotation to", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clusterId", + "description": "The global id of the cluster to add an annotation to", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "startingAt", + "description": "Timestamp indicating starting moment to which the annotation relates", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "endingAt", + "description": "Timestamp indicating ending moment to which the annotation relates", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "dashboardPath", + "description": "The path to a file defining the dashboard on which the annotation should be added", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "description", + "description": "The description of the annotation", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateAnnotationPayload", + "description": "Autogenerated return type of CreateAnnotation", + "fields": [ + { + "name": "annotation", + "description": "The created annotation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "MetricsDashboardAnnotation", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "CreateBranchInput", + "description": "Autogenerated input type of CreateBranch", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "Project full path the branch is associated with", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "name", + "description": "Name of the branch", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "ref", + "description": "Branch name or commit SHA to create branch from", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateBranchPayload", + "description": "Autogenerated return type of CreateBranch", + "fields": [ + { + "name": "branch", + "description": "Branch after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Branch", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateDiffNoteInput", "description": "Autogenerated input type of CreateDiffNote", "fields": null, @@ -1317,7 +3019,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -1539,7 +3241,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -1655,7 +3357,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -1703,6 +3405,148 @@ }, { "kind": "INPUT_OBJECT", + "name": "CreateIterationInput", + "description": "Autogenerated input type of CreateIteration", + "fields": null, + "inputFields": [ + { + "name": "groupPath", + "description": "The target group for the iteration", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "title", + "description": "The title of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "description", + "description": "The description of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "startDate", + "description": "The start date of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "dueDate", + "description": "The end date of the iteration", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateIterationPayload", + "description": "Autogenerated return type of CreateIteration", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "iteration", + "description": "The created iteration", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Iteration", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateNoteInput", "description": "Autogenerated input type of CreateNote", "fields": null, @@ -1781,7 +3625,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -1897,7 +3741,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -2022,6 +3866,24 @@ "defaultValue": null }, { + "name": "uploadedFiles", + "description": "The paths to files uploaded in the snippet description", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", "type": { @@ -2057,7 +3919,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -3679,7 +5541,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -3843,7 +5705,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -4350,7 +6212,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -4452,7 +6314,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -5459,7 +7321,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -7019,7 +8881,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -7696,6 +9558,20 @@ "deprecationReason": null }, { + "name": "iteration", + "description": "Iteration of the issue", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Iteration", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "labels", "description": "Labels of the issue", "args": [ @@ -8550,7 +10426,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -8693,13 +10569,9 @@ "name": "adjacentReferenceId", "description": "The id of the epic_issue or issue that the actual epic or issue is switched with", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } + "kind": "SCALAR", + "name": "ID", + "ofType": null }, "defaultValue": null }, @@ -8707,13 +10579,9 @@ "name": "relativePosition", "description": "The type of the switch, after or before allowed", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "MoveType", - "ofType": null - } + "kind": "ENUM", + "name": "MoveType", + "ofType": null }, "defaultValue": null }, @@ -8802,7 +10670,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -8952,6 +10820,77 @@ "deprecationReason": null }, { + "name": "packageFileRegistries", + "description": "Package file registries of the GeoNode. Available only when feature flag `geo_self_service_framework` is enabled", + "args": [ + { + "name": "ids", + "description": "Filters registries by their ID", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "PackageFileRegistryConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "primary", "description": "Indicates whether this Geo node is the primary", "args": [ @@ -9161,7 +11100,7 @@ }, { "name": "grafanaUrl", - "description": "Url for the Grafana host for the Grafana integration", + "description": "URL for the Grafana host for the Grafana integration", "args": [ ], @@ -10000,6 +11939,99 @@ "deprecationReason": null }, { + "name": "iterations", + "description": "Find iterations", + "args": [ + { + "name": "startDate", + "description": "List items within a time frame where items.start_date is between startDate and endDate parameters (endDate parameter must be present)", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "List items within a time frame where items.end_date is between startDate and endDate parameters (startDate parameter must be present)", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "state", + "description": "Filter iterations by state", + "type": { + "kind": "ENUM", + "name": "IterationState", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "title", + "description": "Fuzzy search by title", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IterationConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "lfsEnabled", "description": "Indicates if Large File Storage (LFS) is enabled for namespace", "args": [ @@ -10062,6 +12094,16 @@ "defaultValue": null }, { + "name": "includeDescendants", + "description": "Return also milestones in all subgroups and subprojects", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -10189,6 +12231,16 @@ "defaultValue": "false" }, { + "name": "hasVulnerabilities", + "description": "Returns only the projects which have vulnerabilities", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "false" + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -10317,29 +12369,41 @@ "args": [ { "name": "startDate", - "description": "List time logs within a time range where the logged date is after start_date parameter.", + "description": "List time logs within a date range where the logged date is equal to or after startDate", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Time", - "ofType": null - } + "kind": "SCALAR", + "name": "Time", + "ofType": null }, "defaultValue": null }, { "name": "endDate", - "description": "List time logs within a time range where the logged date is before end_date parameter.", + "description": "List time logs within a date range where the logged date is equal to or before endDate", "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Time", - "ofType": null - } + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "startTime", + "description": "List time-logs within a time range where the logged time is equal to or after startTime", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "endTime", + "description": "List time-logs within a time range where the logged time is equal to or before endTime", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null }, "defaultValue": null }, @@ -10444,7 +12508,7 @@ }, { "name": "vulnerabilities", - "description": "Vulnerabilities reported on the projects in the group and its subgroups. Available only when feature flag `first_class_vulnerabilities` is enabled", + "description": "Vulnerabilities reported on the projects in the group and its subgroups", "args": [ { "name": "projectId", @@ -10568,6 +12632,87 @@ "deprecationReason": null }, { + "name": "vulnerabilitiesCountByDayAndSeverity", + "description": "Number of vulnerabilities per severity level, per day, for the projects in the group and its subgroups", + "args": [ + { + "name": "startDate", + "description": "First day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "Last day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverityConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "webUrl", "description": "Web URL of the group", "args": [ @@ -10665,6 +12810,86 @@ }, { "kind": "SCALAR", + "name": "ISO8601Date", + "description": "An ISO 8601-encoded date", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "InstanceSecurityDashboard", + "description": null, + "fields": [ + { + "name": "projects", + "description": "Projects selected in Instance Security Dashboard", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ProjectConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "SCALAR", "name": "Int", "description": "Represents non-fractional signed whole numeric values. Int can represent values between -(2^31) and 2^31 - 1.", "fields": null, @@ -11038,6 +13263,20 @@ "deprecationReason": null }, { + "name": "iteration", + "description": "Iteration of the issue", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Iteration", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "labels", "description": "Labels of the issue", "args": [ @@ -11864,7 +14103,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -11994,7 +14233,133 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "IssueSetIterationInput", + "description": "Autogenerated input type of IssueSetIteration", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the issue to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the issue to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iterationId", + "description": "The iteration to assign to the issue.\n", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IssueSetIterationPayload", + "description": "Autogenerated return type of IssueSetIteration", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -12124,7 +14489,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -12203,6 +14568,42 @@ "deprecationReason": null }, { + "name": "PRIORITY_ASC", + "description": "Priority by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PRIORITY_DESC", + "description": "Priority by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "LABEL_PRIORITY_ASC", + "description": "Label priority by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "LABEL_PRIORITY_DESC", + "description": "Label priority by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "MILESTONE_DUE_ASC", + "description": "Milestone due date by ascending order", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "MILESTONE_DUE_DESC", + "description": "Milestone due date by descending order", + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "DUE_DATE_ASC", "description": "Due date by ascending order", "isDeprecated": false, @@ -12265,6 +14666,340 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "Iteration", + "description": "Represents an iteration object.", + "fields": [ + { + "name": "createdAt", + "description": "Timestamp of iteration creation", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "description", + "description": "Description of the iteration", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dueDate", + "description": "Timestamp of the iteration due date", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the iteration", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startDate", + "description": "Timestamp of the iteration start date", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state", + "description": "State of the iteration", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "IterationState", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title", + "description": "Title of the iteration", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Timestamp of last iteration update", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "webPath", + "description": "Web path of the iteration", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "webUrl", + "description": "Web URL of the iteration", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IterationConnection", + "description": "The connection type for Iteration.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "IterationEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Iteration", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "IterationEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Iteration", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "IterationState", + "description": "State of a GitLab iteration", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "upcoming", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "started", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "opened", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "closed", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "all", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "SCALAR", "name": "JSON", "description": "Represents untyped JSON", @@ -12280,6 +15015,20 @@ "description": null, "fields": [ { + "name": "createdAt", + "description": "Timestamp of when the Jira import was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "jiraProjectKey", "description": "Project key for the imported Jira project", "args": [ @@ -12299,7 +15048,7 @@ }, { "name": "scheduledAt", - "description": "Timestamp of when the Jira import was created", + "description": "Timestamp of when the Jira import was scheduled", "args": [ ], @@ -12525,7 +15274,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -12842,6 +15591,35 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "ListLimitMetric", + "description": "List limit metric setting", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "all_metrics", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue_count", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue_weights", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "MarkAsSpamSnippetInput", "description": "Autogenerated input type of MarkAsSpamSnippet", @@ -12897,7 +15675,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -14490,7 +17268,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -14638,7 +17416,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -14768,7 +17546,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -14894,7 +17672,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -15024,7 +17802,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -15154,7 +17932,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -15291,7 +18069,7 @@ "fields": [ { "name": "annotations", - "description": "Annotations added to the dashboard. Will always return `null` if `metrics_dashboard_annotations` feature flag is disabled", + "description": "Annotations added to the dashboard", "args": [ { "name": "from", @@ -15415,7 +18193,7 @@ ], "type": { "kind": "SCALAR", - "name": "String", + "name": "Time", "ofType": null }, "isDeprecated": false, @@ -15461,7 +18239,7 @@ ], "type": { "kind": "SCALAR", - "name": "String", + "name": "Time", "ofType": null }, "isDeprecated": false, @@ -15941,6 +18719,33 @@ "deprecationReason": null }, { + "name": "addProjectToSecurityDashboard", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "AddProjectToSecurityDashboardInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AddProjectToSecurityDashboardPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "adminSidekiqQueuesDeleteJobs", "description": null, "args": [ @@ -15968,6 +18773,114 @@ "deprecationReason": null }, { + "name": "boardListUpdateLimitMetrics", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "BoardListUpdateLimitMetricsInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "BoardListUpdateLimitMetricsPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createAlertIssue", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateAlertIssueInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateAlertIssuePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createAnnotation", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateAnnotationInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateAnnotationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createBranch", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateBranchInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateBranchPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createDiffNote", "description": null, "args": [ @@ -16049,6 +18962,33 @@ "deprecationReason": null }, { + "name": "createIteration", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateIterationInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateIterationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createNote", "description": null, "args": [ @@ -16400,6 +19340,33 @@ "deprecationReason": null }, { + "name": "issueSetIteration", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "IssueSetIterationInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "IssueSetIterationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issueSetWeight", "description": null, "args": [ @@ -16670,6 +19637,33 @@ "deprecationReason": null }, { + "name": "removeProjectFromSecurityDashboard", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "RemoveProjectFromSecurityDashboardInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "RemoveProjectFromSecurityDashboardPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "todoMarkDone", "description": null, "args": [ @@ -16805,6 +19799,33 @@ "deprecationReason": null }, { + "name": "updateAlertStatus", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "UpdateAlertStatusInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UpdateAlertStatusPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updateEpic", "description": null, "args": [ @@ -17155,6 +20176,16 @@ "defaultValue": "false" }, { + "name": "hasVulnerabilities", + "description": "Returns only the projects which have vulnerabilities", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": "false" + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -17997,6 +21028,527 @@ }, { "kind": "OBJECT", + "name": "Package", + "description": "Represents a package", + "fields": [ + { + "name": "createdAt", + "description": "The created date", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "The ID of the package", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "The name of the package", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "packageType", + "description": "The type of the package", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "PackageTypeEnum", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "The update date", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "version", + "description": "The version of the package", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "PackageConnection", + "description": "The connection type for Package.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PackageEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Package", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "PackageEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Package", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "PackageFileRegistry", + "description": "Represents the sync and verification state of a package file", + "fields": [ + { + "name": "createdAt", + "description": "Timestamp when the PackageFileRegistry was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the PackageFileRegistry", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "lastSyncFailure", + "description": "Error message during sync of the PackageFileRegistry", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "lastSyncedAt", + "description": "Timestamp of the most recent successful sync of the PackageFileRegistry", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "packageFileId", + "description": "ID of the PackageFile", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "retryAt", + "description": "Timestamp after which the PackageFileRegistry should be resynced", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "retryCount", + "description": "Number of consecutive failed sync attempts of the PackageFileRegistry", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state", + "description": "Sync state of the PackageFileRegistry", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "RegistryState", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "PackageFileRegistryConnection", + "description": "The connection type for PackageFileRegistry.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PackageFileRegistryEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PackageFileRegistry", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "PackageFileRegistryEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "PackageFileRegistry", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "PackageTypeEnum", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "MAVEN", + "description": "Packages from the maven package manager", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "NPM", + "description": "Packages from the npm package manager", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CONAN", + "description": "Packages from the conan package manager", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "NUGET", + "description": "Packages from the nuget package manager", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PYPI", + "description": "Packages from the pypi package manager", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "COMPOSER", + "description": "Packages from the composer package manager", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "PageInfo", "description": "Information about pagination in a connection.", "fields": [ @@ -18575,6 +22127,182 @@ "description": null, "fields": [ { + "name": "alertManagementAlert", + "description": "A single Alert Management alert of the project", + "args": [ + { + "name": "iid", + "description": "IID of the alert. For example, \"1\"", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "statuses", + "description": "Alerts with the specified statues. For example, [TRIGGERED]", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "AlertManagementStatus", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "sort", + "description": "Sort alerts by this criteria", + "type": { + "kind": "ENUM", + "name": "AlertManagementAlertSort", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "search", + "description": "Search criteria for filtering alerts. This will search on title, description, service, monitoring_tool.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "alertManagementAlertStatusCounts", + "description": "Counts of alerts by status for the project", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlertStatusCountsType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "alertManagementAlerts", + "description": "Alert Management alerts of the project", + "args": [ + { + "name": "iid", + "description": "IID of the alert. For example, \"1\"", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "statuses", + "description": "Alerts with the specified statues. For example, [TRIGGERED]", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "AlertManagementStatus", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "sort", + "description": "Sort alerts by this criteria", + "type": { + "kind": "ENUM", + "name": "AlertManagementAlertSort", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "search", + "description": "Search criteria for filtering alerts. This will search on title, description, service, monitoring_tool.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlertConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "archived", "description": "Indicates the archived status of the project", "args": [ @@ -19723,6 +23451,59 @@ "deprecationReason": null }, { + "name": "packages", + "description": "Packages of the project", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "PackageConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "path", "description": "Path of the project", "args": [ @@ -19852,6 +23633,86 @@ "deprecationReason": null }, { + "name": "release", + "description": "A single release of the project. Available only when feature flag `graphql_release_data` is enabled", + "args": [ + { + "name": "tagName", + "description": "The name of the tag associated to the release", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Release", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "releases", + "description": "Releases of the project. Available only when feature flag `graphql_release_data` is enabled", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "removeSourceBranchAfterMerge", "description": "Indicates if `Delete source branch` option should be enabled by default for all new merge requests of the project", "args": [ @@ -20428,7 +24289,7 @@ }, { "name": "vulnerabilities", - "description": "Vulnerabilities reported on the project. Available only when feature flag `first_class_vulnerabilities` is enabled", + "description": "Vulnerabilities reported on the project", "args": [ { "name": "projectId", @@ -20553,7 +24414,7 @@ }, { "name": "vulnerabilitySeveritiesCount", - "description": "Counts for each severity of vulnerability of the project. Available only when feature flag `first_class_vulnerabilities` is enabled", + "description": "Counts for each severity of vulnerability of the project", "args": [ ], @@ -21736,6 +25597,20 @@ "deprecationReason": null }, { + "name": "instanceSecurityDashboard", + "description": "Fields related to Instance Security Dashboard", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "InstanceSecurityDashboard", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "metadata", "description": "Metadata about GitLab", "args": [ @@ -21804,6 +25679,79 @@ "deprecationReason": null }, { + "name": "projects", + "description": "Find projects visible to the current user", + "args": [ + { + "name": "membership", + "description": "Limit projects that the current user is a member of", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "search", + "description": "Search criteria", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ProjectConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "snippets", "description": "Find Snippets visible to the current user", "args": [ @@ -22048,6 +25996,430 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "vulnerabilitiesCountByDayAndSeverity", + "description": "Number of vulnerabilities per severity level, per day, for the projects on the current user's instance security dashboard", + "args": [ + { + "name": "startDate", + "description": "First day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "endDate", + "description": "Last day for which to fetch vulnerability history", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverityConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "RegistryState", + "description": "State of a Geo registry.", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "PENDING", + "description": "Registry waiting to be synced", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "STARTED", + "description": "Registry currently syncing", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "SYNCED", + "description": "Registry that is synced", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "FAILED", + "description": "Registry that failed to sync", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "Release", + "description": null, + "fields": [ + { + "name": "author", + "description": "User that created the release", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "commit", + "description": "The commit associated with the release", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Commit", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdAt", + "description": "Timestamp of when the release was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "description", + "description": "Description (also known as \"release notes\") of the release", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "descriptionHtml", + "description": "The GitLab Flavored Markdown rendering of `description`", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "milestones", + "description": "Milestones associated to the release", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MilestoneConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the release", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "releasedAt", + "description": "Timestamp of when the release was released", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "tagName", + "description": "Name of the tag associated with the release", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "tagPath", + "description": "Relative web path to the tag associated with the release", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseConnection", + "description": "The connection type for Release.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Release", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Release", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -22141,7 +26513,95 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "RemoveProjectFromSecurityDashboardInput", + "description": "Autogenerated input type of RemoveProjectFromSecurityDashboard", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "ID of the project to remove from the Instance Security Dashboard", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RemoveProjectFromSecurityDashboardPayload", + "description": "Autogenerated return type of RemoveProjectFromSecurityDashboard", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -24587,25 +29047,25 @@ "deprecationReason": null }, { - "name": "YOUTRACK_SERVICE", + "name": "WEBEX_TEAMS_SERVICE", "description": null, "isDeprecated": false, "deprecationReason": null }, { - "name": "GITHUB_SERVICE", + "name": "YOUTRACK_SERVICE", "description": null, "isDeprecated": false, "deprecationReason": null }, { - "name": "JENKINS_SERVICE", + "name": "GITHUB_SERVICE", "description": null, "isDeprecated": false, "deprecationReason": null }, { - "name": "JENKINS_DEPRECATED_SERVICE", + "name": "JENKINS_SERVICE", "description": null, "isDeprecated": false, "deprecationReason": null @@ -24787,7 +29247,7 @@ }, { "name": "id", - "description": "Id of the snippet", + "description": "ID of the snippet", "args": [ ], @@ -25032,6 +29492,20 @@ "deprecationReason": null }, { + "name": "externalStorage", + "description": "Blob external storage", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "mode", "description": "Blob mode", "args": [ @@ -25106,6 +29580,24 @@ "deprecationReason": null }, { + "name": "renderedAsText", + "description": "Shows whether the blob is rendered as text", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "richData", "description": "Blob highlighted data", "args": [ @@ -26214,7 +30706,7 @@ }, { "name": "id", - "description": "Id of the todo", + "description": "ID of the todo", "args": [ ], @@ -26509,7 +31001,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -26658,7 +31150,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -26737,7 +31229,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -26893,7 +31385,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -27035,7 +31527,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -27559,6 +32051,150 @@ }, { "kind": "INPUT_OBJECT", + "name": "UpdateAlertStatusInput", + "description": "Autogenerated input type of UpdateAlertStatus", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the alert to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the alert to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "status", + "description": "The status to set the alert", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "AlertManagementStatus", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "UpdateAlertStatusPayload", + "description": "Autogenerated return type of UpdateAlertStatus", + "fields": [ + { + "name": "alert", + "description": "The alert after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue created after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "UpdateDiffImagePositionInput", "description": null, "fields": null, @@ -27808,7 +32444,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -27916,7 +32552,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -28082,7 +32718,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -28198,7 +32834,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -28334,7 +32970,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -28486,7 +33122,7 @@ }, { "name": "errors", - "description": "Reasons why the mutation failed.", + "description": "Errors encountered during execution of the mutation.", "args": [ ], @@ -28562,6 +33198,24 @@ "deprecationReason": null }, { + "name": "id", + "description": "ID of the user", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "name", "description": "Human-readable name of the user", "args": [ @@ -28671,6 +33325,24 @@ "deprecationReason": null }, { + "name": "state", + "description": "State of the issue", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "todos", "description": "Todos of the user", "args": [ @@ -29100,6 +33772,173 @@ }, { "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverity", + "description": "Represents the number of vulnerabilities for a particular severity on a particular day", + "fields": [ + { + "name": "count", + "description": "Number of vulnerabilities", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "day", + "description": "Date for the count", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "severity", + "description": "Severity of the counted vulnerabilities", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "VulnerabilitySeverity", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverityConnection", + "description": "The connection type for VulnerabilitiesCountByDayAndSeverity.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverityEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverity", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverityEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilitiesCountByDayAndSeverity", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Vulnerability", "description": "Represents a vulnerability.", "fields": [ @@ -29137,13 +33976,13 @@ }, { "name": "location", - "description": "The JSON location metadata for the vulnerability. Its format depends on the type of the security scan that found the vulnerability", + "description": "Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability", "args": [ ], "type": { - "kind": "SCALAR", - "name": "JSON", + "kind": "UNION", + "name": "VulnerabilityLocation", "ofType": null }, "isDeprecated": false, @@ -29372,6 +34211,285 @@ "possibleTypes": null }, { + "kind": "UNION", + "name": "VulnerabilityLocation", + "description": "Represents a vulnerability location. The fields with data will depend on the vulnerability report type", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": [ + { + "kind": "OBJECT", + "name": "VulnerabilityLocationContainerScanning", + "ofType": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationDast", + "ofType": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationDependencyScanning", + "ofType": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationSast", + "ofType": null + } + ] + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationContainerScanning", + "description": "Represents the location of a vulnerability found by a container security scan", + "fields": [ + { + "name": "dependency", + "description": "Dependency containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerableDependency", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "image", + "description": "Name of the vulnerable container image", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "operatingSystem", + "description": "Operating system that runs on the vulnerable container image", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationDast", + "description": "Represents the location of a vulnerability found by a DAST scan", + "fields": [ + { + "name": "hostname", + "description": "Domain name of the vulnerable request", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "param", + "description": "Query parameter for the URL on which the vulnerability occurred", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "path", + "description": "URL path and query string of the vulnerable request", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "requestMethod", + "description": "HTTP method of the vulnerable request", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationDependencyScanning", + "description": "Represents the location of a vulnerability found by a dependency security scan", + "fields": [ + { + "name": "dependency", + "description": "Dependency containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerableDependency", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "file", + "description": "Path to the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationSast", + "description": "Represents the location of a vulnerability found by a SAST scan", + "fields": [ + { + "name": "endLine", + "description": "Number of the last relevant line in the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "file", + "description": "Path to the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startLine", + "description": "Number of the first relevant line in the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerableClass", + "description": "Class containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerableMethod", + "description": "Method containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "VulnerabilityPermissions", "description": "Check permissions for the current user on a vulnerability", @@ -29744,6 +34862,74 @@ }, { "kind": "OBJECT", + "name": "VulnerableDependency", + "description": "Represents a vulnerable dependency. Used in vulnerability location data", + "fields": [ + { + "name": "package", + "description": "The package associated with the vulnerable dependency", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerablePackage", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "version", + "description": "The version of the vulnerable dependency", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerablePackage", + "description": "Represents a vulnerable package. Used in vulnerability dependency data", + "fields": [ + { + "name": "name", + "description": "The name of the vulnerable package", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "__Directive", "description": "A Directive provides a way to describe alternate runtime execution and type validation behavior in a GraphQL document.\n\nIn some cases, you need to provide options to alter GraphQL's execution behavior in ways field arguments will not suffice, such as conditionally including or skipping a field. Directives provide this by describing additional information to the executor.", "fields": [ diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 4d3d77ba35f..4164c26e751 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -24,7 +24,17 @@ Autogenerated return type of AddAwardEmoji | --- | ---- | ---------- | | `awardEmoji` | AwardEmoji | The award emoji after mutation | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + +## AddProjectToSecurityDashboardPayload + +Autogenerated return type of AddProjectToSecurityDashboard + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `project` | Project | Project that was added to the Instance Security Dashboard | ## AdminSidekiqQueuesDeleteJobsPayload @@ -33,9 +43,44 @@ Autogenerated return type of AdminSidekiqQueuesDeleteJobs | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `result` | DeleteJobsResponse | Information about the status of the deletion request | +## AlertManagementAlert + +Describes an alert from the project's Alert Management + +| Name | Type | Description | +| --- | ---- | ---------- | +| `createdAt` | Time | Timestamp the alert was created | +| `description` | String | Description of the alert | +| `details` | JSON | Alert details | +| `endedAt` | Time | Timestamp the alert ended | +| `eventCount` | Int | Number of events of this alert | +| `hosts` | String! => Array | List of hosts the alert came from | +| `iid` | ID! | Internal ID of the alert | +| `issueIid` | ID | Internal ID of the GitLab issue attached to the alert | +| `monitoringTool` | String | Monitoring tool the alert came from | +| `service` | String | Service the alert came from | +| `severity` | AlertManagementSeverity | Severity of the alert | +| `startedAt` | Time | Timestamp the alert was raised | +| `status` | AlertManagementStatus | Status of the alert | +| `title` | String | Title of the alert | +| `updatedAt` | Time | Timestamp the alert was last updated | + +## AlertManagementAlertStatusCountsType + +Represents total number of alerts for the represented categories + +| Name | Type | Description | +| --- | ---- | ---------- | +| `acknowledged` | Int | Number of alerts with status ACKNOWLEDGED for the project | +| `all` | Int | Total number of alerts for the project | +| `ignored` | Int | Number of alerts with status IGNORED for the project | +| `open` | Int | Number of alerts with status TRIGGERED or ACKNOWLEDGED for the project | +| `resolved` | Int | Number of alerts with status RESOLVED for the project | +| `triggered` | Int | Number of alerts with status TRIGGERED for the project | + ## AwardEmoji An emoji awarded by a user. @@ -79,6 +124,41 @@ Represents a project or group board | `name` | String | Name of the board | | `weight` | Int | Weight of the board | +## BoardList + +Represents a list for an issue board + +| Name | Type | Description | +| --- | ---- | ---------- | +| `assignee` | User | Assignee in the list | +| `collapsed` | Boolean | Indicates if list is collapsed for this user | +| `id` | ID! | ID (global ID) of the list | +| `label` | Label | Label of the list | +| `limitMetric` | ListLimitMetric | The current limit metric for the list | +| `listType` | String! | Type of the list | +| `maxIssueCount` | Int | Maximum number of issues in the list | +| `maxIssueWeight` | Int | Maximum weight of issues in the list | +| `milestone` | Milestone | Milestone of the list | +| `position` | Int | Position of list within the board | +| `title` | String! | Title of the list | + +## BoardListUpdateLimitMetricsPayload + +Autogenerated return type of BoardListUpdateLimitMetrics + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `list` | BoardList | The updated list | + +## Branch + +| Name | Type | Description | +| --- | ---- | ---------- | +| `commit` | Commit | Commit for the branch | +| `name` | String! | Name of the branch | + ## Commit | Name | Type | Description | @@ -94,8 +174,40 @@ Represents a project or group board | `sha` | String! | SHA1 ID of the commit | | `signatureHtml` | String | Rendered HTML of the commit signature | | `title` | String | Title of the commit message | +| `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | | `webUrl` | String! | Web URL of the commit | +## CreateAlertIssuePayload + +Autogenerated return type of CreateAlertIssue + +| Name | Type | Description | +| --- | ---- | ---------- | +| `alert` | AlertManagementAlert | The alert after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue created after mutation | + +## CreateAnnotationPayload + +Autogenerated return type of CreateAnnotation + +| Name | Type | Description | +| --- | ---- | ---------- | +| `annotation` | MetricsDashboardAnnotation | The created annotation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + +## CreateBranchPayload + +Autogenerated return type of CreateBranch + +| Name | Type | Description | +| --- | ---- | ---------- | +| `branch` | Branch | Branch after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ## CreateDiffNotePayload Autogenerated return type of CreateDiffNote @@ -103,7 +215,7 @@ Autogenerated return type of CreateDiffNote | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | ## CreateEpicPayload @@ -114,7 +226,7 @@ Autogenerated return type of CreateEpic | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `epic` | Epic | The created epic | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | ## CreateImageDiffNotePayload @@ -123,9 +235,19 @@ Autogenerated return type of CreateImageDiffNote | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | +## CreateIterationPayload + +Autogenerated return type of CreateIteration + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `iteration` | Iteration | The created iteration | + ## CreateNotePayload Autogenerated return type of CreateNote @@ -133,7 +255,7 @@ Autogenerated return type of CreateNote | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | ## CreateRequirementPayload @@ -143,7 +265,7 @@ Autogenerated return type of CreateRequirement | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `requirement` | Requirement | The requirement after mutation | ## CreateSnippetPayload @@ -153,7 +275,7 @@ Autogenerated return type of CreateSnippet | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | ## DeleteJobsResponse @@ -228,7 +350,7 @@ Autogenerated return type of DesignManagementDelete | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `version` | DesignVersion | The new version in which the designs are deleted | ## DesignManagementUploadPayload @@ -239,7 +361,7 @@ Autogenerated return type of DesignManagementUpload | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `designs` | Design! => Array | The designs that were uploaded by the mutation | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `skippedDesigns` | Design! => Array | Any designs that were skipped from the upload due to there being no change to their content since their last version | ## DesignVersion @@ -259,7 +381,7 @@ Autogenerated return type of DestroyNote | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | ## DestroySnippetPayload @@ -269,7 +391,7 @@ Autogenerated return type of DestroySnippet | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | ## DetailedStatus @@ -324,7 +446,7 @@ Autogenerated return type of DismissVulnerability | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `vulnerability` | Vulnerability | The vulnerability after dismissal | ## Environment @@ -389,7 +511,7 @@ Autogenerated return type of EpicAddIssue | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `epic` | Epic | The epic after mutation | | `epicIssue` | EpicIssue | The epic-issue relation | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | ## EpicDescendantCount @@ -443,6 +565,7 @@ Relationship between an epic and an issue | `healthStatus` | HealthStatus | Current health status. Returns null if `save_issuable_health_status` feature flag is disabled. | | `id` | ID | Global ID of the epic-issue relation | | `iid` | ID! | Internal ID of the issue | +| `iteration` | Iteration | Iteration of the issue | | `milestone` | Milestone | Milestone of the issue | | `reference` | String! | Internal reference of the issue. Returned in shortened format by default | | `relationPath` | String | URI path of the epic-issue relation | @@ -485,7 +608,7 @@ Autogenerated return type of EpicSetSubscription | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `epic` | Epic | The epic after mutation | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | ## EpicTreeReorderPayload @@ -494,7 +617,7 @@ Autogenerated return type of EpicTreeReorder | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | ## GeoNode @@ -521,7 +644,7 @@ Autogenerated return type of EpicTreeReorder | --- | ---- | ---------- | | `createdAt` | Time! | Timestamp of the issue's creation | | `enabled` | Boolean! | Indicates whether Grafana integration is enabled | -| `grafanaUrl` | String! | Url for the Grafana host for the Grafana integration | +| `grafanaUrl` | String! | URL for the Grafana host for the Grafana integration | | `id` | ID! | Internal ID of the Grafana integration | | `token` **{warning-solid}** | String! | **Deprecated:** Plain text token has been masked for security reasons. Deprecated in 12.7 | | `updatedAt` | Time! | Timestamp of the issue's last activity | @@ -582,6 +705,7 @@ Autogenerated return type of EpicTreeReorder | `epic` | Epic | Epic to which this issue belongs | | `healthStatus` | HealthStatus | Current health status. Returns null if `save_issuable_health_status` feature flag is disabled. | | `iid` | ID! | Internal ID of the issue | +| `iteration` | Iteration | Iteration of the issue | | `milestone` | Milestone | Milestone of the issue | | `reference` | String! | Internal reference of the issue. Returned in shortened format by default | | `relativePosition` | Int | Relative position of the issue (used for positioning in epic tree and issue boards) | @@ -622,7 +746,7 @@ Autogenerated return type of IssueSetConfidential | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | ## IssueSetDueDatePayload @@ -632,7 +756,17 @@ Autogenerated return type of IssueSetDueDate | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + +## IssueSetIterationPayload + +Autogenerated return type of IssueSetIteration + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | ## IssueSetWeightPayload @@ -642,15 +776,33 @@ Autogenerated return type of IssueSetWeight | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | +## Iteration + +Represents an iteration object. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `createdAt` | Time! | Timestamp of iteration creation | +| `description` | String | Description of the iteration | +| `dueDate` | Time | Timestamp of the iteration due date | +| `id` | ID! | ID of the iteration | +| `startDate` | Time | Timestamp of the iteration start date | +| `state` | IterationState! | State of the iteration | +| `title` | String! | Title of the iteration | +| `updatedAt` | Time! | Timestamp of last iteration update | +| `webPath` | String! | Web path of the iteration | +| `webUrl` | String! | Web URL of the iteration | + ## JiraImport | Name | Type | Description | | --- | ---- | ---------- | +| `createdAt` | Time | Timestamp of when the Jira import was created | | `jiraProjectKey` | String! | Project key for the imported Jira project | -| `scheduledAt` | Time | Timestamp of when the Jira import was created | +| `scheduledAt` | Time | Timestamp of when the Jira import was scheduled | | `scheduledBy` | User | User that started the Jira import | ## JiraImportStartPayload @@ -660,7 +812,7 @@ Autogenerated return type of JiraImportStart | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `jiraImport` | JiraImport | The Jira import data after mutation | ## JiraService @@ -688,7 +840,7 @@ Autogenerated return type of MarkAsSpamSnippet | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | ## MergeRequest @@ -767,7 +919,7 @@ Autogenerated return type of MergeRequestSetAssignees | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | ## MergeRequestSetLabelsPayload @@ -777,7 +929,7 @@ Autogenerated return type of MergeRequestSetLabels | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | ## MergeRequestSetLockedPayload @@ -787,7 +939,7 @@ Autogenerated return type of MergeRequestSetLocked | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | ## MergeRequestSetMilestonePayload @@ -797,7 +949,7 @@ Autogenerated return type of MergeRequestSetMilestone | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | ## MergeRequestSetSubscriptionPayload @@ -807,7 +959,7 @@ Autogenerated return type of MergeRequestSetSubscription | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | ## MergeRequestSetWipPayload @@ -817,7 +969,7 @@ Autogenerated return type of MergeRequestSetWip | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `mergeRequest` | MergeRequest | The merge request after mutation | ## Metadata @@ -838,10 +990,10 @@ Autogenerated return type of MergeRequestSetWip | Name | Type | Description | | --- | ---- | ---------- | | `description` | String | Description of the annotation | -| `endingAt` | String | Timestamp marking end of annotated time span | +| `endingAt` | Time | Timestamp marking end of annotated time span | | `id` | ID! | ID of the annotation | | `panelId` | String | ID of a dashboard panel to which the annotation should be scoped | -| `startingAt` | String | Timestamp marking start of annotated time span | +| `startingAt` | Time | Timestamp marking start of annotated time span | ## Milestone @@ -905,6 +1057,34 @@ Represents a milestone. | `readNote` | Boolean! | Indicates the user can perform `read_note` on this resource | | `resolveNote` | Boolean! | Indicates the user can perform `resolve_note` on this resource | +## Package + +Represents a package + +| Name | Type | Description | +| --- | ---- | ---------- | +| `createdAt` | Time! | The created date | +| `id` | ID! | The ID of the package | +| `name` | String! | The name of the package | +| `packageType` | PackageTypeEnum! | The type of the package | +| `updatedAt` | Time! | The update date | +| `version` | String | The version of the package | + +## PackageFileRegistry + +Represents the sync and verification state of a package file + +| Name | Type | Description | +| --- | ---- | ---------- | +| `createdAt` | Time | Timestamp when the PackageFileRegistry was created | +| `id` | ID! | ID of the PackageFileRegistry | +| `lastSyncFailure` | String | Error message during sync of the PackageFileRegistry | +| `lastSyncedAt` | Time | Timestamp of the most recent successful sync of the PackageFileRegistry | +| `packageFileId` | ID! | ID of the PackageFile | +| `retryAt` | Time | Timestamp after which the PackageFileRegistry should be resynced | +| `retryCount` | Int | Number of consecutive failed sync attempts of the PackageFileRegistry | +| `state` | RegistryState | Sync state of the PackageFileRegistry | + ## PageInfo Information about pagination in a connection. @@ -947,6 +1127,8 @@ Information about pagination in a connection. | Name | Type | Description | | --- | ---- | ---------- | +| `alertManagementAlert` | AlertManagementAlert | A single Alert Management alert of the project | +| `alertManagementAlertStatusCounts` | AlertManagementAlertStatusCountsType | Counts of alerts by status for the project | | `archived` | Boolean | Indicates the archived status of the project | | `autocloseReferencedIssues` | Boolean | Indicates if issues referenced by merge requests and commits within the default branch are closed automatically | | `avatarUrl` | String | URL to avatar image file of the project | @@ -980,6 +1162,7 @@ Information about pagination in a connection. | `path` | String! | Path of the project | | `printingMergeRequestLinkEnabled` | Boolean | Indicates if a link to create or view a merge request should display after a push to Git repositories of the project from the command line | | `publicJobs` | Boolean | Indicates if there is public access to pipelines and job details of the project, including output logs and artifacts | +| `release` | Release | A single release of the project. Available only when feature flag `graphql_release_data` is enabled | | `removeSourceBranchAfterMerge` | Boolean | Indicates if `Delete source branch` option should be enabled by default for all new merge requests of the project | | `repository` | Repository | Git repository of the project | | `requestAccessEnabled` | Boolean | Indicates if users can request member access to the project | @@ -998,7 +1181,7 @@ Information about pagination in a connection. | `tagList` | String | List of project topics (not Git tags) | | `userPermissions` | ProjectPermissions! | Permissions for the current user on the resource | | `visibility` | String | Visibility of the project | -| `vulnerabilitySeveritiesCount` | VulnerabilitySeveritiesCount | Counts for each severity of vulnerability of the project. Available only when feature flag `first_class_vulnerabilities` is enabled | +| `vulnerabilitySeveritiesCount` | VulnerabilitySeveritiesCount | Counts for each severity of vulnerability of the project | | `webUrl` | String | Web URL of the project | | `wikiEnabled` | Boolean | Indicates if Wikis are enabled for the current user | @@ -1061,6 +1244,20 @@ Information about pagination in a connection. | `storageSize` | Float! | Storage size of the project | | `wikiSize` | Float | Wiki size of the project | +## Release + +| Name | Type | Description | +| --- | ---- | ---------- | +| `author` | User | User that created the release | +| `commit` | Commit | The commit associated with the release | +| `createdAt` | Time | Timestamp of when the release was created | +| `description` | String | Description (also known as "release notes") of the release | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `name` | String | Name of the release | +| `releasedAt` | Time | Timestamp of when the release was released | +| `tagName` | String! | Name of the tag associated with the release | +| `tagPath` | String | Relative web path to the tag associated with the release | + ## RemoveAwardEmojiPayload Autogenerated return type of RemoveAwardEmoji @@ -1069,7 +1266,16 @@ Autogenerated return type of RemoveAwardEmoji | --- | ---- | ---------- | | `awardEmoji` | AwardEmoji | The award emoji after mutation | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + +## RemoveProjectFromSecurityDashboardPayload + +Autogenerated return type of RemoveProjectFromSecurityDashboard + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | ## Repository @@ -1256,7 +1462,7 @@ Represents a snippet entry | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | | `fileName` | String | File Name of the snippet | | `httpUrlToRepo` | String | HTTP URL to the snippet repository | -| `id` | ID! | Id of the snippet | +| `id` | ID! | ID of the snippet | | `project` | Project | The project the snippet is associated with | | `rawUrl` | String! | Raw URL of the snippet | | `sshUrlToRepo` | String | SSH URL to the snippet repository | @@ -1273,11 +1479,13 @@ Represents the snippet blob | Name | Type | Description | | --- | ---- | ---------- | | `binary` | Boolean! | Shows whether the blob is binary | +| `externalStorage` | String | Blob external storage | | `mode` | String | Blob mode | | `name` | String | Blob name | | `path` | String | Blob path | | `plainData` | String | Blob plain highlighted data | | `rawPath` | String! | Blob raw content endpoint path | +| `renderedAsText` | Boolean! | Shows whether the blob is rendered as text | | `richData` | String | Blob highlighted data | | `richViewer` | SnippetBlobViewer | Blob content rich viewer | | `simpleViewer` | SnippetBlobViewer! | Blob content simple viewer | @@ -1351,7 +1559,7 @@ Representing a todo entry | `body` | String! | Body of the todo | | `createdAt` | Time! | Timestamp this todo was created | | `group` | Group | Group this todo is associated with | -| `id` | ID! | Id of the todo | +| `id` | ID! | ID of the todo | | `project` | Project | The project this todo is associated with | | `state` | TodoStateEnum! | State of the todo | | `targetType` | TodoTargetEnum! | Target type of the todo | @@ -1363,7 +1571,7 @@ Autogenerated return type of TodoMarkDone | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `todo` | Todo! | The requested todo | ## TodoRestoreManyPayload @@ -1373,7 +1581,7 @@ Autogenerated return type of TodoRestoreMany | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `updatedIds` | ID! => Array | The ids of the updated todo items | ## TodoRestorePayload @@ -1383,7 +1591,7 @@ Autogenerated return type of TodoRestore | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `todo` | Todo! | The requested todo | ## TodosMarkAllDonePayload @@ -1393,7 +1601,7 @@ Autogenerated return type of TodosMarkAllDone | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `updatedIds` | ID! => Array | Ids of the updated todos | ## ToggleAwardEmojiPayload @@ -1404,7 +1612,7 @@ Autogenerated return type of ToggleAwardEmoji | --- | ---- | ---------- | | `awardEmoji` | AwardEmoji | The award emoji after mutation | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `toggledOn` | Boolean! | Indicates the status of the emoji. True if the toggle awarded the emoji, and false if the toggle removed the emoji. | ## Tree @@ -1427,6 +1635,17 @@ Represents a directory | `type` | EntryType! | Type of tree entry | | `webUrl` | String | Web URL for the tree entry (directory) | +## UpdateAlertStatusPayload + +Autogenerated return type of UpdateAlertStatus + +| Name | Type | Description | +| --- | ---- | ---------- | +| `alert` | AlertManagementAlert | The alert after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue created after mutation | + ## UpdateEpicPayload Autogenerated return type of UpdateEpic @@ -1435,7 +1654,7 @@ Autogenerated return type of UpdateEpic | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | | `epic` | Epic | The epic after mutation | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | ## UpdateImageDiffNotePayload @@ -1444,7 +1663,7 @@ Autogenerated return type of UpdateImageDiffNote | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | ## UpdateIssuePayload @@ -1454,7 +1673,7 @@ Autogenerated return type of UpdateIssue | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue after mutation | ## UpdateNotePayload @@ -1464,7 +1683,7 @@ Autogenerated return type of UpdateNote | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | ## UpdateRequirementPayload @@ -1474,7 +1693,7 @@ Autogenerated return type of UpdateRequirement | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `requirement` | Requirement | The requirement after mutation | ## UpdateSnippetPayload @@ -1484,7 +1703,7 @@ Autogenerated return type of UpdateSnippet | Name | Type | Description | | --- | ---- | ---------- | | `clientMutationId` | String | A unique identifier for the client performing the mutation. | -| `errors` | String! => Array | Reasons why the mutation failed. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | ## User @@ -1492,7 +1711,9 @@ Autogenerated return type of UpdateSnippet | Name | Type | Description | | --- | ---- | ---------- | | `avatarUrl` | String | URL of the user's avatar | +| `id` | ID! | ID of the user | | `name` | String! | Human-readable name of the user | +| `state` | String! | State of the issue | | `userPermissions` | UserPermissions! | Permissions for the current user on the resource | | `username` | String! | Username of the user. Unique within this instance of GitLab | | `webUrl` | String! | Web URL of the user | @@ -1503,6 +1724,16 @@ Autogenerated return type of UpdateSnippet | --- | ---- | ---------- | | `createSnippet` | Boolean! | Indicates the user can perform `create_snippet` on this resource | +## VulnerabilitiesCountByDayAndSeverity + +Represents the number of vulnerabilities for a particular severity on a particular day + +| Name | Type | Description | +| --- | ---- | ---------- | +| `count` | Int | Number of vulnerabilities | +| `day` | ISO8601Date | Date for the count | +| `severity` | VulnerabilitySeverity | Severity of the counted vulnerabilities | + ## Vulnerability Represents a vulnerability. @@ -1511,7 +1742,7 @@ Represents a vulnerability. | --- | ---- | ---------- | | `description` | String | Description of the vulnerability | | `id` | ID! | GraphQL ID of the vulnerability | -| `location` | JSON | The JSON location metadata for the vulnerability. Its format depends on the type of the security scan that found the vulnerability | +| `location` | VulnerabilityLocation | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability | | `project` | Project | The project on which the vulnerability was found | | `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST) | | `severity` | VulnerabilitySeverity | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) | @@ -1520,6 +1751,48 @@ Represents a vulnerability. | `userPermissions` | VulnerabilityPermissions! | Permissions for the current user on the resource | | `vulnerabilityPath` | String | URL to the vulnerability's details page | +## VulnerabilityLocationContainerScanning + +Represents the location of a vulnerability found by a container security scan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `dependency` | VulnerableDependency | Dependency containing the vulnerability | +| `image` | String | Name of the vulnerable container image | +| `operatingSystem` | String | Operating system that runs on the vulnerable container image | + +## VulnerabilityLocationDast + +Represents the location of a vulnerability found by a DAST scan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `hostname` | String | Domain name of the vulnerable request | +| `param` | String | Query parameter for the URL on which the vulnerability occurred | +| `path` | String | URL path and query string of the vulnerable request | +| `requestMethod` | String | HTTP method of the vulnerable request | + +## VulnerabilityLocationDependencyScanning + +Represents the location of a vulnerability found by a dependency security scan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `dependency` | VulnerableDependency | Dependency containing the vulnerability | +| `file` | String | Path to the vulnerable file | + +## VulnerabilityLocationSast + +Represents the location of a vulnerability found by a SAST scan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `endLine` | String | Number of the last relevant line in the vulnerable file | +| `file` | String | Path to the vulnerable file | +| `startLine` | String | Number of the first relevant line in the vulnerable file | +| `vulnerableClass` | String | Class containing the vulnerability | +| `vulnerableMethod` | String | Method containing the vulnerability | + ## VulnerabilityPermissions Check permissions for the current user on a vulnerability @@ -1547,3 +1820,20 @@ Represents vulnerability counts by severity | `low` | Int | Number of vulnerabilities of LOW severity of the project | | `medium` | Int | Number of vulnerabilities of MEDIUM severity of the project | | `unknown` | Int | Number of vulnerabilities of UNKNOWN severity of the project | + +## VulnerableDependency + +Represents a vulnerable dependency. Used in vulnerability location data + +| Name | Type | Description | +| --- | ---- | ---------- | +| `package` | VulnerablePackage | The package associated with the vulnerable dependency | +| `version` | String | The version of the vulnerable dependency | + +## VulnerablePackage + +Represents a vulnerable package. Used in vulnerability dependency data + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String | The name of the vulnerable package | diff --git a/doc/api/group_badges.md b/doc/api/group_badges.md index 6d5961a7eb8..5b7164cdd4d 100644 --- a/doc/api/group_badges.md +++ b/doc/api/group_badges.md @@ -7,7 +7,7 @@ Badges support placeholders that will be replaced in real time in both the link and image URL. The allowed placeholders are: - **%{project_path}**: will be replaced by the project path. -- **%{project_id}**: will be replaced by the project id. +- **%{project_id}**: will be replaced by the project ID. - **%{default_branch}**: will be replaced by the project default branch. - **%{commit_sha}**: will be replaced by the last project's commit SHA. diff --git a/doc/api/group_import_export.md b/doc/api/group_import_export.md index 50e8bd9dcf2..355ecbfb98f 100644 --- a/doc/api/group_import_export.md +++ b/doc/api/group_import_export.md @@ -1,6 +1,6 @@ # Group Import/Export API -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20353) in GitLab 12.8 as an experimental feature. May change in future releases. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20353) in GitLab 12.8. Group Import/Export allows you to export group structure and import it to a new location. When used with [Project Import/Export](project_import_export.md), you can preserve connections with @@ -25,7 +25,7 @@ POST /groups/:id/export | Attribute | Type | Required | Description | | --------- | -------------- | -------- | ---------------------------------------- | -| `id` | integer/string | yes | ID of the groupd owned by the authenticated user | +| `id` | integer/string | yes | ID of the group owned by the authenticated user | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/export @@ -41,7 +41,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab Download the finished export. -```text +```plaintext GET /groups/:id/export/download ``` @@ -66,7 +66,7 @@ returns either: ## Import a file -```text +```plaintext POST /groups/import ``` diff --git a/doc/api/group_milestones.md b/doc/api/group_milestones.md index 7e2a6987208..10445acf881 100644 --- a/doc/api/group_milestones.md +++ b/doc/api/group_milestones.md @@ -1,6 +1,6 @@ # Group milestones API -> [Introduced][ce-12819] in GitLab 9.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12819) in GitLab 9.5. ## List group milestones @@ -21,10 +21,10 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ------ | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | -| `iids[]` | integer array | optional | Return only the milestones having the given `iid` | -| `state` | string | optional | Return only `active` or `closed` milestones | -| `title` | string | optional | Return only the milestones having the given `title` | -| `search` | string | optional | Return only milestones with a title or description matching the provided string | +| `iids[]` | integer array | no | Return only the milestones having the given `iid` | +| `state` | string | no | Return only `active` or `closed` milestones | +| `title` | string | no | Return only the milestones having the given `title` | +| `search` | string | no | Return only milestones with a title or description matching the provided string | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/milestones @@ -44,7 +44,8 @@ Example Response: "start_date": "2013-11-10", "state": "active", "updated_at": "2013-10-02T09:24:18Z", - "created_at": "2013-10-02T09:24:18Z" + "created_at": "2013-10-02T09:24:18Z", + "web_url": "https://gitlab.com/groups/gitlab-org/-/milestones/42" } ] ``` @@ -59,8 +60,10 @@ GET /groups/:id/milestones/:milestone_id Parameters: -- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user -- `milestone_id` (required) - The ID of the group milestone +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `milestone_id` | integer | yes | The ID of the group milestone | ## Create new milestone @@ -72,11 +75,13 @@ POST /groups/:id/milestones Parameters: -- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user -- `title` (required) - The title of a milestone -- `description` (optional) - The description of the milestone -- `due_date` (optional) - The due date of the milestone -- `start_date` (optional) - The start date of the milestone +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `title` | string | yes | The title of a milestone | +| `description` | string | no | The description of the milestone | +| `due_date` | date | no | The due date of the milestone, in YYYY-MM-DD format (ISO 8601) | +| `start_date` | date | no | The start date of the milestone, in YYYY-MM-DD format (ISO 8601) | ## Edit milestone @@ -88,13 +93,15 @@ PUT /groups/:id/milestones/:milestone_id Parameters: -- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user -- `milestone_id` (required) - The ID of a group milestone -- `title` (optional) - The title of a milestone -- `description` (optional) - The description of a milestone -- `due_date` (optional) - The due date of the milestone -- `start_date` (optional) - The start date of the milestone -- `state_event` (optional) - The state event of the milestone (close|activate) +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `milestone_id` | integer | yes | The ID of a group milestone | +| `title` | string | no | The title of a milestone | +| `description` | string | no | The description of a milestone | +| `due_date` | date | no | The due date of the milestone, in YYYY-MM-DD format (ISO 8601) | +| `start_date` | date | no | The start date of the milestone, in YYYY-MM-DD format (ISO 8601) | +| `state_event` | string | no | The state event of the milestone _(`close` or `activate`)_ | ## Delete group milestone @@ -106,8 +113,10 @@ DELETE /groups/:id/milestones/:milestone_id Parameters: -- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user -- `milestone_id` (required) - The ID of the group's milestone +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `milestone_id` | integer | yes | The ID of the group's milestone | ## Get all issues assigned to a single milestone @@ -119,8 +128,10 @@ GET /groups/:id/milestones/:milestone_id/issues Parameters: -- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user -- `milestone_id` (required) - The ID of a group milestone +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `milestone_id` | integer | yes | The ID of a group milestone | ## Get all merge requests assigned to a single milestone @@ -132,10 +143,10 @@ GET /groups/:id/milestones/:milestone_id/merge_requests Parameters: -- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user -- `milestone_id` (required) - The ID of a group milestone - -[ce-12819]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12819 +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `milestone_id` | integer | yes | The ID of a group milestone | ## Get all burndown chart events for a single milestone **(STARTER)** @@ -149,5 +160,7 @@ GET /groups/:id/milestones/:milestone_id/burndown_events Parameters: -- `id` (required) - The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user -- `milestone_id` (required) - The ID of a group milestone +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `milestone_id` | integer | yes | The ID of a group milestone | diff --git a/doc/api/groups.md b/doc/api/groups.md index 34c01766d06..bc7bff2964b 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -109,7 +109,7 @@ GET /groups?custom_attributes[key]=value&custom_attributes[other_key]=other_valu ## List a group's subgroups -> [Introduced][ce-15142] in GitLab 10.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15142) in GitLab 10.3. Get a list of visible direct subgroups in this group. When accessed without authentication, only public groups are returned. @@ -241,9 +241,142 @@ Example response: ``` NOTE: **Note:** - To distinguish between a project in the group and a project shared to the group, the `namespace` attribute can be used. When a project has been shared to the group, its `namespace` will be different from the group the request is being made for. +## List a group's shared projects + +Get a list of projects shared to this group. When accessed without authentication, only public shared projects are returned. + +By default, this request returns 20 results at a time because the API results [are paginated](README.md#pagination). + +```plaintext +GET /groups/:id/projects/shared +``` + +Parameters: + +| Attribute | Type | Required | Description | +| ----------------------------- | -------------- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `archived` | boolean | no | Limit by archived status | +| `visibility` | string | no | Limit by visibility `public`, `internal`, or `private` | +| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | +| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | +| `search` | string | no | Return list of authorized projects matching the search criteria | +| `simple` | boolean | no | Return only the ID, URL, name, and path of each project | +| `starred` | boolean | no | Limit by projects starred by the current user | +| `with_issues_enabled` | boolean | no | Limit by projects with issues feature enabled. Default is `false` | +| `with_merge_requests_enabled` | boolean | no | Limit by projects with merge requests feature enabled. Default is `false` | +| `min_access_level` | integer | no | Limit to projects where current user has at least this [access level](members.md) | +| `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | + +Example response: + +```json +[ + { + "id":8, + "description":"Shared project for Html5 Boilerplate", + "name":"Html5 Boilerplate", + "name_with_namespace":"H5bp / Html5 Boilerplate", + "path":"html5-boilerplate", + "path_with_namespace":"h5bp/html5-boilerplate", + "created_at":"2020-04-27T06:13:22.642Z", + "default_branch":"master", + "tag_list":[ + + ], + "ssh_url_to_repo":"ssh://git@gitlab.com/h5bp/html5-boilerplate.git", + "http_url_to_repo":"http://gitlab.com/h5bp/html5-boilerplate.git", + "web_url":"http://gitlab.com/h5bp/html5-boilerplate", + "readme_url":"http://gitlab.com/h5bp/html5-boilerplate/-/blob/master/README.md", + "avatar_url":null, + "star_count":0, + "forks_count":4, + "last_activity_at":"2020-04-27T06:13:22.642Z", + "namespace":{ + "id":28, + "name":"H5bp", + "path":"h5bp", + "kind":"group", + "full_path":"h5bp", + "parent_id":null, + "avatar_url":null, + "web_url":"http://gitlab.com/groups/h5bp" + }, + "_links":{ + "self":"http://gitlab.com/api/v4/projects/8", + "issues":"http://gitlab.com/api/v4/projects/8/issues", + "merge_requests":"http://gitlab.com/api/v4/projects/8/merge_requests", + "repo_branches":"http://gitlab.com/api/v4/projects/8/repository/branches", + "labels":"http://gitlab.com/api/v4/projects/8/labels", + "events":"http://gitlab.com/api/v4/projects/8/events", + "members":"http://gitlab.com/api/v4/projects/8/members" + }, + "empty_repo":false, + "archived":false, + "visibility":"public", + "resolve_outdated_diff_discussions":false, + "container_registry_enabled":true, + "container_expiration_policy":{ + "cadence":"7d", + "enabled":true, + "keep_n":null, + "older_than":null, + "name_regex":null, + "name_regex_keep":null, + "next_run_at":"2020-05-04T06:13:22.654Z" + }, + "issues_enabled":true, + "merge_requests_enabled":true, + "wiki_enabled":true, + "jobs_enabled":true, + "snippets_enabled":true, + "can_create_merge_request_in":true, + "issues_access_level":"enabled", + "repository_access_level":"enabled", + "merge_requests_access_level":"enabled", + "forking_access_level":"enabled", + "wiki_access_level":"enabled", + "builds_access_level":"enabled", + "snippets_access_level":"enabled", + "pages_access_level":"enabled", + "emails_disabled":null, + "shared_runners_enabled":true, + "lfs_enabled":true, + "creator_id":1, + "import_status":"failed", + "open_issues_count":10, + "ci_default_git_depth":50, + "public_jobs":true, + "build_timeout":3600, + "auto_cancel_pending_pipelines":"enabled", + "build_coverage_regex":null, + "ci_config_path":null, + "shared_with_groups":[ + { + "group_id":24, + "group_name":"Commit451", + "group_full_path":"Commit451", + "group_access_level":30, + "expires_at":null + } + ], + "only_allow_merge_if_pipeline_succeeds":false, + "request_access_enabled":true, + "only_allow_merge_if_all_discussions_are_resolved":false, + "remove_source_branch_after_merge":true, + "printing_merge_request_link_enabled":true, + "merge_method":"merge", + "suggestion_commit_message":null, + "auto_devops_enabled":true, + "auto_devops_deploy_strategy":"continuous", + "autoclose_referenced_issues":true, + "repository_storage":"default" + } +] +``` + ## Details of a group Get all details of a group. This endpoint can be accessed without authentication @@ -259,7 +392,11 @@ Parameters: | ------------------------ | -------------- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user. | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only). | -| `with_projects` | boolean | no | Include details from projects that belong to the specified group (defaults to `true`). (Deprecated, [will be removed in 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). To get the details of all projects within a group, use the [list a group's projects endpoint](#list-a-groups-projects).) | +| `with_projects` | boolean | no | Include details from projects that belong to the specified group (defaults to `true`). (Deprecated, [will be removed in API v5](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). To get the details of all projects within a group, use the [list a group's projects endpoint](#list-a-groups-projects).) | + +NOTE: **Note:** +The `projects` and `shared_projects` attributes in the response are deprecated and will be [removed in API v5](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). +To get the details of all projects within a group, use either the [list a group's projects](#list-a-groups-projects) or the [list a group's shared projects](#list-a-groups-shared-projects) endpoint. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4 @@ -290,7 +427,7 @@ Example response: "file_template_project_id": 1, "parent_id": null, "created_at": "2020-01-15T12:36:29.590Z", - "projects": [ + "projects": [ // Deprecated and will be removed in API v5 { "id": 7, "description": "Voluptas veniam qui et beatae voluptas doloremque explicabo facilis.", @@ -368,7 +505,7 @@ Example response: "request_access_enabled": false } ], - "shared_projects": [ + "shared_projects": [ // Deprecated and will be removed in API v5 { "id": 8, "description": "Velit eveniet provident fugiat saepe eligendi autem.", @@ -571,6 +708,10 @@ PUT /groups/:id | `shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Pipeline minutes quota for this group. | | `extra_shared_runners_minutes_limit` | integer | no | **(STARTER ONLY)** Extra pipeline minutes quota for this group. | +NOTE: **Note:** +The `projects` and `shared_projects` attributes in the response are deprecated and will be [removed in API v5](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). +To get the details of all projects within a group, use either the [list a group's projects](#list-a-groups-projects) or the [list a group's shared projects](#list-a-groups-shared-projects) endpoint. + ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5?name=Experimental" ``` @@ -582,10 +723,6 @@ This endpoint returns: and later. To get the details of all projects within a group, use the [list a group's projects endpoint](#list-a-groups-projects) instead. -NOTE: **Note:** - -The `projects` and `shared_projects` attributes [will be deprecated in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). To get the details of all projects within a group, use the [list a group's projects endpoint](#list-a-groups-projects) instead. - Example response: ```json @@ -603,7 +740,7 @@ Example response: "file_template_project_id": 1, "parent_id": null, "created_at": "2020-01-15T12:36:29.590Z", - "projects": [ + "projects": [ // Deprecated and will be removed in API v5 { "id": 9, "description": "foo", @@ -944,8 +1081,6 @@ And to switch pages add: /groups?per_page=100&page=2 ``` -[ce-15142]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15142 - ## Group badges Read more in the [Group Badges](group_badges.md) documentation. diff --git a/doc/api/import.md b/doc/api/import.md index 8db8dc7eea4..9640ba19cf9 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -12,8 +12,8 @@ POST /import/github |------------|---------|----------|---------------------| | `personal_access_token` | string | yes | GitHub personal access token | | `repo_id` | integer | yes | GitHub repository ID | -| `new_name` | string | no | New repo name | -| `target_namespace` | string | yes | Namespace to import repo into | +| `new_name` | string | no | New repository name | +| `target_namespace` | string | yes | Namespace to import repository into | ```shell curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --data "personal_access_token=abc123&repo_id=12345&target_namespace=root" https://gitlab.example.com/api/v4/import/github diff --git a/doc/api/instance_level_ci_variables.md b/doc/api/instance_level_ci_variables.md new file mode 100644 index 00000000000..d0871fdf4a7 --- /dev/null +++ b/doc/api/instance_level_ci_variables.md @@ -0,0 +1,162 @@ +# Instance-level CI/CD variables API + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14108) in GitLab 13.0 +> - It's deployed behind a feature flag, enabled by default. +> - It's enabled on GitLab.com. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-instance-level-cicd-variables-core-only). **(CORE ONLY)** + +## List all instance variables + +Get the list of all instance-level variables. + +```plaintext +GET /admin/ci/variables +``` + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/admin/ci/variables" +``` + +```json +[ + { + "key": "TEST_VARIABLE_1", + "variable_type": "env_var", + "value": "TEST_1", + "protected": false, + "masked": false + }, + { + "key": "TEST_VARIABLE_2", + "variable_type": "env_var", + "value": "TEST_2", + "protected": false, + "masked": false + } +] +``` + +## Show instance variable details + +Get the details of a specific instance-level variable. + +```plaintext +GET /admin/ci/variables/:key +``` + +| Attribute | Type | required | Description | +|-----------|---------|----------|-----------------------| +| `key` | string | yes | The `key` of a variable | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/admin/ci/variables/TEST_VARIABLE_1" +``` + +```json +{ + "key": "TEST_VARIABLE_1", + "variable_type": "env_var", + "value": "TEST_1", + "protected": false, + "masked": false +} +``` + +## Create instance variable + +Create a new instance-level variable. + +NOTE: **Note:** +The maximum number of instance-level variables is [planned to be 25](https://gitlab.com/gitlab-org/gitlab/-/issues/216097). + +```plaintext +POST /admin/ci/variables +``` + +| Attribute | Type | required | Description | +|-----------------|---------|----------|-----------------------| +| `key` | string | yes | The `key` of a variable. Max 255 characters, only `A-Z`, `a-z`, `0-9`, and `_` are allowed. | +| `value` | string | yes | The `value` of a variable. | +| `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file`. | +| `protected` | boolean | no | Whether the variable is protected. | +| `masked` | boolean | no | Whether the variable is masked. | + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/admin/ci/variables" --form "key=NEW_VARIABLE" --form "value=new value" +``` + +```json +{ + "key": "NEW_VARIABLE", + "value": "new value", + "variable_type": "env_var", + "protected": false, + "masked": false +} +``` + +## Update instance variable + +Update an instance-level variable. + +```plaintext +PUT /admin/ci/variables/:key +``` + +| Attribute | Type | required | Description | +|-----------------|---------|----------|-------------------------| +| `key` | string | yes | The `key` of a variable. | +| `value` | string | yes | The `value` of a variable. | +| `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file`. | +| `protected` | boolean | no | Whether the variable is protected. | +| `masked` | boolean | no | Whether the variable is masked. | + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/admin/ci/variables/NEW_VARIABLE" --form "value=updated value" +``` + +```json +{ + "key": "NEW_VARIABLE", + "value": "updated value", + "variable_type": "env_var", + "protected": true, + "masked": true +} +``` + +## Remove instance variable + +Remove an instance-level variable. + +```plaintext +DELETE /admin/ci/variables/:key +``` + +| Attribute | Type | required | Description | +|-----------|---------|----------|-------------------------| +| `key` | string | yes | The `key` of a variable | + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/admin/ci/variables/VARIABLE_1" +``` + +### Enable or disable instance-level CI/CD variables **(CORE ONLY)** + +Instance-level CI/CD variables is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:ci_instance_level_variables) +``` + +To enable it: + +```ruby +Feature.enable(:ci_instance_level_variables) +``` diff --git a/doc/api/issues.md b/doc/api/issues.md index 14f81d7d327..8e5882c4d4e 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -53,7 +53,7 @@ GET /issues?confidential=true | `author_id` | integer | no | Return issues created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13004) in GitLab 9.5)_ | | `author_username` | string | no | Return issues created by the given `username`. Similar to `author_id` and mutually exclusive with `author_id`. | | `assignee_id` | integer | no | Return issues assigned to the given user `id`. Mutually exclusive with `assignee_username`. `None` returns unassigned issues. `Any` returns issues with an assignee. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13004) in GitLab 9.5)_ | -| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In CE version `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | +| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In GitLab CE `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | | `my_reaction_emoji` | string | no | Return issues reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016) in GitLab 10.0)_ | | `weight` **(STARTER)** | integer | no | Return issues with the specified `weight`. `None` returns issues with no weight assigned. `Any` returns issues with a weight assigned. | | `iids[]` | integer array | no | Return only the issues having the given `iid` | @@ -67,6 +67,7 @@ GET /issues?confidential=true | `updated_before` | datetime | no | Return issues updated on or before the given time | | `confidential` | boolean | no | Filter confidential or public issues. | | `not` | Hash | no | Return issues that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji`, `search`, `in` | +| `non_archived` | boolean | no | Return issues only from non-archived projects. If `false`, response will return issues from both archived and non-archived projects. Default is `true`. _(Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/197170))_ | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/issues @@ -210,7 +211,7 @@ GET /groups/:id/issues?confidential=true | `author_id` | integer | no | Return issues created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13004) in GitLab 9.5)_ | | `author_username` | string | no | Return issues created by the given `username`. Similar to `author_id` and mutually exclusive with `author_id`. | | `assignee_id` | integer | no | Return issues assigned to the given user `id`. Mutually exclusive with `assignee_username`. `None` returns unassigned issues. `Any` returns issues with an assignee. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13004) in GitLab 9.5)_ | -| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In CE version `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | +| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In GitLab CE `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | | `my_reaction_emoji` | string | no | Return issues reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016) in GitLab 10.0)_ | | `weight` **(STARTER)** | integer | no | Return issues with the specified `weight`. `None` returns issues with no weight assigned. `Any` returns issues with a weight assigned. | | `order_by` | string | no | Return issues ordered by `created_at`, `updated_at`, `priority`, `due_date`, `relative_position`, `label_priority`, `milestone_due`, `popularity`, `weight` fields. Default is `created_at` | @@ -365,7 +366,7 @@ GET /projects/:id/issues?confidential=true | `author_id` | integer | no | Return issues created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13004) in GitLab 9.5)_ | | `author_username` | string | no | Return issues created by the given `username`. Similar to `author_id` and mutually exclusive with `author_id`. | | `assignee_id` | integer | no | Return issues assigned to the given user `id`. Mutually exclusive with `assignee_username`. `None` returns unassigned issues. `Any` returns issues with an assignee. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13004) in GitLab 9.5)_ | -| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In CE version `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | +| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In GitLab CE `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | | `my_reaction_emoji` | string | no | Return issues reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016) in GitLab 10.0)_ | | `weight` **(STARTER)** | integer | no | Return issues with the specified `weight`. `None` returns issues with no weight assigned. `Any` returns issues with a weight assigned. | | `order_by` | string | no | Return issues ordered by `created_at`, `updated_at`, `priority`, `due_date`, `relative_position`, `label_priority`, `milestone_due`, `popularity`, `weight` fields. Default is `created_at` | @@ -630,7 +631,7 @@ the `epic` property: **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. -**Note**: The `epic_iid` attribute is deprecated and [will be removed in 13.0](https://gitlab.com/gitlab-org/gitlab/issues/35157). +**Note**: The `epic_iid` attribute is deprecated and [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/issues/35157). Please use `iid` of the `epic` attribute instead. ## New issue @@ -657,7 +658,7 @@ POST /projects/:id/issues | `discussion_to_resolve` | string | no | The ID of a discussion to resolve. This will fill in the issue with a default description and mark the discussion as resolved. Use in combination with `merge_request_to_resolve_discussions_of`. | | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. | | `epic_id` **(ULTIMATE)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in 13.0](https://gitlab.com/gitlab-org/gitlab/issues/35157)) | +| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/issues/35157)) | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues?title=Issues%20with%20auth&labels=bug @@ -773,7 +774,7 @@ PUT /projects/:id/issues/:issue_iid | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. 0 | | `discussion_locked` | boolean | no | Flag indicating if the issue's discussion is locked. If the discussion is locked only project members can add or edit comments. | | `epic_id` **(ULTIMATE)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in 13.0](https://gitlab.com/gitlab-org/gitlab/issues/35157)) | +| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/issues/35157)) | ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues/85?state_event=close diff --git a/doc/api/issues_statistics.md b/doc/api/issues_statistics.md index 699eda174d2..8db99e93f79 100644 --- a/doc/api/issues_statistics.md +++ b/doc/api/issues_statistics.md @@ -34,7 +34,7 @@ GET /issues_statistics?confidential=true | `author_id` | integer | no | Return issues created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. | | `author_username` | string | no | Return issues created by the given `username`. Similar to `author_id` and mutually exclusive with `author_id`. | | `assignee_id` | integer | no | Return issues assigned to the given user `id`. Mutually exclusive with `assignee_username`. `None` returns unassigned issues. `Any` returns issues with an assignee. | -| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In CE version `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | +| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In GitLab CE `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | | `my_reaction_emoji` | string | no | Return issues reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | | `iids[]` | integer array | no | Return only the issues having the given `iid` | | `search` | string | no | Search issues against their `title` and `description` | @@ -92,7 +92,7 @@ GET /groups/:id/issues_statistics?confidential=true | `author_id` | integer | no | Return issues created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. | | `author_username` | string | no | Return issues created by the given `username`. Similar to `author_id` and mutually exclusive with `author_id`. | | `assignee_id` | integer | no | Return issues assigned to the given user `id`. Mutually exclusive with `assignee_username`. `None` returns unassigned issues. `Any` returns issues with an assignee. | -| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In CE version `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | +| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In GitLab CE `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | | `my_reaction_emoji` | string | no | Return issues reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | | `search` | string | no | Search group issues against their `title` and `description` | | `created_after` | datetime | no | Return issues created on or after the given time | @@ -148,7 +148,7 @@ GET /projects/:id/issues_statistics?confidential=true | `author_id` | integer | no | Return issues created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. | | `author_username` | string | no | Return issues created by the given `username`. Similar to `author_id` and mutually exclusive with `author_id`. | | `assignee_id` | integer | no | Return issues assigned to the given user `id`. Mutually exclusive with `assignee_username`. `None` returns unassigned issues. `Any` returns issues with an assignee. | -| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In CE version `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | +| `assignee_username` | string array | no | Return issues assigned to the given `username`. Similar to `assignee_id` and mutually exclusive with `assignee_id`. In GitLab CE `assignee_username` array should only contain a single value or an invalid parameter error will be returned otherwise. | | `my_reaction_emoji` | string | no | Return issues reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | | `search` | string | no | Search project issues against their `title` and `description` | | `created_after` | datetime | no | Return issues created on or after the given time | diff --git a/doc/api/jobs.md b/doc/api/jobs.md index 4827eafd790..c06dc56407c 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -345,7 +345,7 @@ Example of response > **Notes**: > -> - [Introduced][ce-2893] in GitLab 8.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2893) in GitLab 8.5. > - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) > in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. @@ -399,13 +399,11 @@ Possible response status codes: | 200 | Serves the artifacts file. | | 404 | Build not found or no artifacts.| -[ce-2893]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2893 - ## Download the artifacts archive > **Notes**: > -> - [Introduced][ce-5347] in GitLab 8.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5347) in GitLab 8.10. > - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) > in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. @@ -466,8 +464,6 @@ Possible response status codes: | 200 | Serves the artifacts file. | | 404 | Build not found or no artifacts.| -[ce-5347]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5347 - ## Download a single artifact file by job ID > Introduced in GitLab 10.0 @@ -547,8 +543,8 @@ GET /projects/:id/jobs/:job_id/trace | Attribute | Type | Required | Description | |-----------|----------------|----------|------------------------------------------------------------------------------------------------------------------| -| id | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | -| job_id | integer | yes | ID of a job. | +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `job_id` | integer | yes | ID of a job. | ```shell curl --location --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs/8/trace" diff --git a/doc/api/keys.md b/doc/api/keys.md index d4eb1161a97..81ebd70be52 100644 --- a/doc/api/keys.md +++ b/doc/api/keys.md @@ -4,7 +4,7 @@ Get SSH key with user by ID of an SSH key. Note only administrators can lookup SSH key with user by ID of an SSH key. -```text +```plaintext GET /keys/:id ``` @@ -63,7 +63,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/a You can search for a user that owns a specific SSH key. Note only administrators can lookup SSH key with the fingerprint of an SSH key. -```text +```plaintext GET /keys ``` diff --git a/doc/api/labels.md b/doc/api/labels.md index e3f367daaca..3ced7da8ed5 100644 --- a/doc/api/labels.md +++ b/doc/api/labels.md @@ -7,6 +7,8 @@ The `description_html` - was added to response JSON in [GitLab 12.7](https://git Get all labels for a given project. +By default, this request returns 20 results at a time because the API results [are paginated](README.md#pagination). + ```plaintext GET /projects/:id/labels ``` @@ -109,7 +111,7 @@ GET /projects/:id/labels/:label_id | Attribute | Type | Required | Description | | --------- | ------- | -------- | --------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `label_id` | integer or string | yes | The ID or title of a group's label. | +| `label_id` | integer or string | yes | The ID or title of a project's label. | | `include_ancestor_groups` | boolean | no | Include ancestor groups. Defaults to `true`. | ```shell diff --git a/doc/api/lint.md b/doc/api/lint.md index 4ecce92df26..f0e4ad5655a 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -1,6 +1,6 @@ # Validate the `.gitlab-ci.yml` (API) -> [Introduced][ce-5953] in GitLab 8.12. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5953) in GitLab 8.12. Checks if your `.gitlab-ci.yml` file is valid. @@ -47,5 +47,3 @@ Example responses: "error": "content is missing" } ``` - -[ce-5953]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5953 diff --git a/doc/api/managed_licenses.md b/doc/api/managed_licenses.md index 66125d23a82..13eb3a3fea7 100644 --- a/doc/api/managed_licenses.md +++ b/doc/api/managed_licenses.md @@ -90,7 +90,7 @@ Example response: ## Delete a managed license -Deletes a managed license with a given id. +Deletes a managed license with a given ID. ```plaintext DELETE /projects/:id/managed_licenses/:managed_license_id diff --git a/doc/api/members.md b/doc/api/members.md index e9131e2d4c3..afeda7780d7 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -282,6 +282,78 @@ Example response: } ``` +### Set override flag for a member of a group + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4875) in GitLab 12.10. + +By default, the access level of LDAP group members is set to the value specified +by LDAP through Group Sync. You can allow access level overrides by calling this endpoint. + +```plaintext +POST /groups/:id/members/:user_id/override +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `user_id` | integer | yes | The user ID of the member | + +```bash +curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/:id/members/:user_id/override +``` + +Example response: + +```json +{ + "id": 1, + "username": "raymond_smith", + "name": "Raymond Smith", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/c2525a7f58ae3776070e44c106c48e15?s=80&d=identicon", + "web_url": "http://192.168.1.8:3000/root", + "expires_at": "2012-10-22T14:13:35Z", + "access_level": 40, + "override": true +} +``` + +### Remove override for a member of a group + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4875) in GitLab 12.10. + +Sets the override flag to false and allows LDAP Group Sync to reset the access +level to the LDAP-prescribed value. + +```plaintext +DELETE /groups/:id/members/:user_id/override +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | +| `user_id` | integer | yes | The user ID of the member | + +```bash +curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/:id/members/:user_id/override +``` + +Example response: + +```json +{ + "id": 1, + "username": "raymond_smith", + "name": "Raymond Smith", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/c2525a7f58ae3776070e44c106c48e15?s=80&d=identicon", + "web_url": "http://192.168.1.8:3000/root", + "expires_at": "2012-10-22T14:13:35Z", + "access_level": 40, + "override": false +} +``` + ## Remove a member from a group or project Removes a user from a group or project. diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 84f5e41496c..c07e52451d2 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -51,7 +51,7 @@ POST /projects/:id/approvals | `approvals_before_merge` | integer | no | How many approvals are required before an MR can be merged. Deprecated in 12.0 in favor of Approval Rules API. | | `reset_approvals_on_push` | boolean | no | Reset approvals on a new push | | `disable_overriding_approvers_per_merge_request` | boolean | no | Allow/Disallow overriding approvers per MR | -| `merge_requests_author_approval` | boolean | no | Allow/Disallow authors from self approving merge requests; `true` means authors cannot self approve | +| `merge_requests_author_approval` | boolean | no | Allow/Disallow authors from self approving merge requests; `true` means authors can self approve | | `merge_requests_disable_committers_approval` | boolean | no | Allow/Disallow committers from self approving merge requests | | `require_password_to_approve` | boolean | no | Require approver to enter a password in order to authenticate before adding the approval | diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index b0ffeae2b55..3834bb6fee3 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -12,7 +12,7 @@ NOTE: **Note** ## List merge requests -> [Introduced][ce-13060] in GitLab 9.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 9.5. Get all merge requests the authenticated user has access to. By default it returns only merge requests created by the current user. To @@ -47,17 +47,18 @@ Parameters: | `view` | string | no | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request | | `labels` | string | no | Return merge requests matching a comma separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. `No+Label` (Deprecated) lists all merge requests with no labels. Predefined names are case-insensitive. | | `with_labels_details` | boolean | no | If `true`, response will return more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. Introduced in [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21413) | +| `with_merge_status_recheck` | boolean | no | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) | | `created_after` | datetime | no | Return merge requests created on or after the given time | | `created_before` | datetime | no | Return merge requests created on or before the given time | | `updated_after` | datetime | no | Return merge requests updated on or after the given time | | `updated_before` | datetime | no | Return merge requests updated on or before the given time | | `scope` | string | no | Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`. Defaults to `created_by_me`<br> For versions before 11.0, use the now deprecated `created-by-me` or `assigned-to-me` scopes instead. | | `author_id` | integer | no | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me` -| `author_username` | string | no | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. _([Introduced][ce-13060] in GitLab 12.10)_ | | +| `author_username` | string | no | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 12.10)_ | | | `assignee_id` | integer | no | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. | | `approver_ids` **(STARTER)** | integer array | no | Returns merge requests which have specified all the users with the given `id`s as individual approvers. `None` returns merge requests without approvers. `Any` returns merge requests with an approver. | | `approved_by_ids` **(STARTER)** | integer array | no | Returns merge requests which have been approved by all the users with the given `id`s (Max: 5). `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | -| `my_reaction_emoji` | string | no | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced][ce-14016] in GitLab 10.0)_ | +| `my_reaction_emoji` | string | no | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016) in GitLab 10.0)_ | | `source_branch` | string | no | Return merge requests with the given source branch | | `target_branch` | string | no | Return merge requests with the given target branch | | `search` | string | no | Search merge requests against their `title` and `description` | @@ -65,6 +66,13 @@ Parameters: | `wip` | string | no | Filter merge requests against their `wip` status. `yes` to return *only* WIP merge requests, `no` to return *non* WIP merge requests | NOTE: **Note:** +[Starting in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890), +listing merge requests may not proactively update the `merge_status` field +(which also affects the `has_conflicts` field), as this can be an expensive +operation. If you are interested in the value of these fields from this +endpoint, set the `with_merge_status_recheck` parameter to `true` in the query. + +NOTE: **Note:** [Starting in GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/29984), when `async_merge_request_check_mergeability` feature flag is enabled, the mergeability (`merge_status`) of each merge request will be checked @@ -227,17 +235,18 @@ Parameters: | `view` | string | no | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request | | `labels` | string | no | Return merge requests matching a comma separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. `No+Label` (Deprecated) lists all merge requests with no labels. Predefined names are case-insensitive. | | `with_labels_details` | boolean | no | If `true`, response will return more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. Introduced in [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21413) | +| `with_merge_status_recheck` | boolean | no | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) | | `created_after` | datetime | no | Return merge requests created on or after the given time | | `created_before` | datetime | no | Return merge requests created on or before the given time | | `updated_after` | datetime | no | Return merge requests updated on or after the given time | | `updated_before` | datetime | no | Return merge requests updated on or before the given time | -| `scope` | string | no | Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`.<br> For versions before 11.0, use the now deprecated `created-by-me` or `assigned-to-me` scopes instead.<br> _([Introduced][ce-13060] in GitLab 9.5. [Changed to snake_case][ce-18935] in GitLab 11.0)_ | -| `author_id` | integer | no | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. _([Introduced][ce-13060] in GitLab 9.5)_ -| `author_username` | string | no | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. _([Introduced][ce-13060] in GitLab 12.10)_ | | -| `assignee_id` | integer | no | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. _([Introduced][ce-13060] in GitLab 9.5)_ | +| `scope` | string | no | Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`.<br> For versions before 11.0, use the now deprecated `created-by-me` or `assigned-to-me` scopes instead.<br> _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 9.5. [Changed to snake_case](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18935) in GitLab 11.0)_ | +| `author_id` | integer | no | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 9.5)_ +| `author_username` | string | no | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 12.10)_ | | +| `assignee_id` | integer | no | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 9.5)_ | | `approver_ids` **(STARTER)** | integer array | no | Returns merge requests which have specified all the users with the given `id`s as individual approvers. `None` returns merge requests without approvers. `Any` returns merge requests with an approver. | | `approved_by_ids` **(STARTER)** | integer array | no | Returns merge requests which have been approved by all the users with the given `id`s (Max: 5). `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | -| `my_reaction_emoji` | string | no | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced][ce-14016] in GitLab 10.0)_ | +| `my_reaction_emoji` | string | no | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016) in GitLab 10.0)_ | | `source_branch` | string | no | Return merge requests with the given source branch | | `target_branch` | string | no | Return merge requests with the given target branch | | `search` | string | no | Search merge requests against their `title` and `description` | @@ -390,17 +399,18 @@ Parameters: | `view` | string | no | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request | | `labels` | string | no | Return merge requests matching a comma separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. `No+Label` (Deprecated) lists all merge requests with no labels. Predefined names are case-insensitive. | | `with_labels_details` | boolean | no | If `true`, response will return more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. Introduced in [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21413)| +| `with_merge_status_recheck` | boolean | no | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) | | `created_after` | datetime | no | Return merge requests created on or after the given time | | `created_before` | datetime | no | Return merge requests created on or before the given time | | `updated_after` | datetime | no | Return merge requests updated on or after the given time | | `updated_before` | datetime | no | Return merge requests updated on or before the given time | | `scope` | string | no | Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`.<br> | -| `author_id` | integer | no | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. _([Introduced][ce-13060] in GitLab 9.5)_ -| `author_username` | string | no | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. _([Introduced][ce-13060] in GitLab 12.10)_ | | -| `assignee_id` | integer | no | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. _([Introduced][ce-13060] in GitLab 9.5)_ | +| `author_id` | integer | no | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 9.5)_ +| `author_username` | string | no | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 12.10)_ | | +| `assignee_id` | integer | no | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060) in GitLab 9.5)_ | | `approver_ids` **(STARTER)** | integer array | no | Returns merge requests which have specified all the users with the given `id`s as individual approvers. `None` returns merge requests without approvers. `Any` returns merge requests with an approver. | | `approved_by_ids` **(STARTER)** | integer array | no | Returns merge requests which have been approved by all the users with the given `id`s (Max: 5). `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | -| `my_reaction_emoji` | string | no | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced][ce-14016] in GitLab 10.0)_ | +| `my_reaction_emoji` | string | no | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016) in GitLab 10.0)_ | | `source_branch` | string | no | Return merge requests with the given source branch | | `target_branch` | string | no | Return merge requests with the given target branch | | `search` | string | no | Search merge requests against their `title` and `description` | @@ -871,7 +881,7 @@ Parameters: ## List MR pipelines -> [Introduced][ce-15454] in GitLab 10.5.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15454) in GitLab 10.5.0. Get a list of merge request pipelines. @@ -972,7 +982,7 @@ POST /projects/:id/merge_requests | `assignee_id` | integer | no | Assignee user ID | | `assignee_ids` | integer array | no | The ID of the user(s) to assign the MR to. Set to `0` or provide an empty value to unassign all assignees. | | `description` | string | no | Description of MR. Limited to 1,048,576 characters. | -| `target_project_id` | integer | no | The target project (numeric id) | +| `target_project_id` | integer | no | The target project (numeric ID) | | `labels` | string | no | Labels for MR as a comma-separated list | | `milestone_id` | integer | no | The global ID of a milestone | | `remove_source_branch` | boolean | no | Flag indicating if a merge request should remove the source branch when merging | @@ -1130,6 +1140,8 @@ PUT /projects/:id/merge_requests/:merge_request_iid | `assignee_ids` | integer array | no | The ID of the user(s) to assign the MR to. Set to `0` or provide an empty value to unassign all assignees. | | `milestone_id` | integer | no | The global ID of a milestone to assign the merge request to. Set to `0` or provide an empty value to unassign a milestone.| | `labels` | string | no | Comma-separated label names for a merge request. Set to an empty string to unassign all labels. | +| `add_labels` | string | no | Comma-separated label names to add to a merge request. | +| `remove_labels` | string | no | Comma-separated label names to remove from a merge request. | | `description` | string | no | Description of MR. Limited to 1,048,576 characters. | | `state_event` | string | no | New state (close/reopen) | | `remove_source_branch` | boolean | no | Flag indicating if a merge request should remove the source branch when merging | @@ -2435,11 +2447,6 @@ Example response: } ``` -[ce-13060]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13060 -[ce-14016]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016 -[ce-15454]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15454 -[ce-18935]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18935 - ## Approvals **(STARTER)** For approvals, please see [Merge Request Approvals](merge_request_approvals.md) diff --git a/doc/api/metrics_dashboard_annotations.md b/doc/api/metrics_dashboard_annotations.md index d8a018fe6c3..09187a096ef 100644 --- a/doc/api/metrics_dashboard_annotations.md +++ b/doc/api/metrics_dashboard_annotations.md @@ -4,22 +4,16 @@ Metrics dashboard annotations allow you to indicate events on your graphs at a single point in time or over a timespan. -## Enable the metrics dashboard annotations API - -The `:metrics_dashboard_annotations` feature flag is disabled by default. -To turn on this API, ask a GitLab administrator with Rails console -access to run the following command: - -```ruby -Feature.enable(:metrics_dashboard_annotations) -``` - ## Create a new annotation ```plaintext POST /environments/:id/metrics_dashboard/annotations/ +POST /clusters/:id/metrics_dashboard/annotations/ ``` +NOTE: **Note:** +The value of `dashboard_path` will be treated as a CGI-escaped path, and automatically unescaped. + Parameters: | Attribute | Type | Required | Description | diff --git a/doc/api/metrics_user_starred_dashboards.md b/doc/api/metrics_user_starred_dashboards.md new file mode 100644 index 00000000000..dd9144d1319 --- /dev/null +++ b/doc/api/metrics_user_starred_dashboards.md @@ -0,0 +1,61 @@ +# User-starred metrics dashboards API + +The starred dashboard feature makes navigating to frequently-used dashboards easier +by displaying favorited dashboards at the top of the select list. + +## Add a star to a dashboard + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31316) in GitLab 13.0. + +```plaintext +POST /projects/:id/metrics/user_starred_dashboards +``` + +Parameters: + +| Attribute | Type | Required | Description | +|:---------------|:---------------|:---------|:-----------------------------------------------------------------------------| +| `dashboard_path` | string | yes | URL-encoded path to file defining the dashboard which should be marked as favorite. | + +```shell +curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/projects/20/metrics/user_starred_dashboards \ + --data-urlencode "dashboard_path=config/prometheus/dashboards/common_metrics.yml" +``` + +Example Response: + +```json +{ + "id": 5, + "dashboard_path": "config/prometheus/common_metrics.yml", + "user_id": 1, + "project_id": 20 +} +``` + +## Remove a star from a dashboard + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31892) in GitLab 13.0. + +```plaintext +DELETE /projects/:id/metrics/user_starred_dashboards +``` + +Parameters: + +| Attribute | Type | Required | Description | +|:---------------|:---------------|:---------|:-----------------------------------------------------------------------------| +| `dashboard_path` | string | no | URL-encoded path to file defining the dashboard which should no longer be marked as favorite. When not supplied all dashboards within given projects will be removed from favorites. | + +```shell +curl --request DELETE --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/projects/20/metrics/user_starred_dashboards \ + --data-urlencode "dashboard_path=config/prometheus/dashboards/common_metrics.yml" +``` + +Example Response: + +```json +{ + "deleted_rows": 1 +} +``` diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index 959773b217d..a146fdd0d0c 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -4,7 +4,7 @@ This document covers using the [OAuth2](https://oauth.net/2/) protocol to allow other services to access GitLab resources on user's behalf. If you want GitLab to be an OAuth authentication service provider to sign into -other services, see the [OAuth2 provider](../integration/oauth_provider.md) +other services, see the [OAuth2 authentication service provider](../integration/oauth_provider.md) documentation. This functionality is based on the [doorkeeper Ruby gem](https://github.com/doorkeeper-gem/doorkeeper). diff --git a/doc/api/packages.md b/doc/api/packages.md index 8671de006d2..784343d29fd 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -68,6 +68,7 @@ GET /groups/:id/packages | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, `type`, or `project_path`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | | `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi` or `nuget`. (_Introduced in GitLab 12.9_) | +| `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30980) in GitLab 13.0_) ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/packages?exclude_subgroups=true @@ -187,7 +188,27 @@ Example response: "name": "Administrator", "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon" } - } + }, + "versions": [ + { + "id":2, + "version":"2.0-SNAPSHOT", + "created_at":"2020-04-28T04:42:11.573Z", + "pipeline": { + "id": 234, + "status": "pending", + "ref": "new-pipeline", + "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", + "web_url": "https://example.com/foo/bar/pipelines/58", + "created_at": "2016-08-11T11:28:34.085Z", + "updated_at": "2016-08-11T11:32:35.169Z", + "user": { + "name": "Administrator", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon" + } + } + } + ] } ``` diff --git a/doc/api/pages.md b/doc/api/pages.md index 627c1b284f4..db39ab04d9d 100644 --- a/doc/api/pages.md +++ b/doc/api/pages.md @@ -8,7 +8,7 @@ The GitLab Pages feature must be enabled to use these endpoints. Find out more a Remove pages. The user must have admin privileges. -```text +```plaintext DELETE /projects/:id/pages ``` diff --git a/doc/api/pages_domains.md b/doc/api/pages_domains.md index 8a047afc3b0..43bb5a9b774 100644 --- a/doc/api/pages_domains.md +++ b/doc/api/pages_domains.md @@ -8,7 +8,7 @@ The GitLab Pages feature must be enabled to use these endpoints. Find out more a Get a list of all pages domains. The user must have admin permissions. -```text +```plaintext GET /pages/domains ``` @@ -35,7 +35,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap Get a list of project pages domains. The user must have permissions to view pages domains. -```text +```plaintext GET /projects/:id/pages/domains ``` @@ -71,7 +71,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap Get a single project pages domain. The user must have permissions to view pages domains. -```text +```plaintext GET /projects/:id/pages/domains/:domain ``` @@ -113,7 +113,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap Creates a new pages domain. The user must have permissions to create new pages domains. -```text +```plaintext POST /projects/:id/pages/domains ``` @@ -155,7 +155,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain Updates an existing project pages domain. The user must have permissions to change an existing pages domains. -```text +```plaintext PUT /projects/:id/pages/domains/:domain ``` @@ -225,7 +225,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certifi Deletes an existing project pages domain. -```text +```plaintext DELETE /projects/:id/pages/domains/:domain ``` diff --git a/doc/api/pipeline_schedules.md b/doc/api/pipeline_schedules.md index 859e88d0945..36a18411ceb 100644 --- a/doc/api/pipeline_schedules.md +++ b/doc/api/pipeline_schedules.md @@ -54,7 +54,7 @@ GET /projects/:id/pipeline_schedules/:pipeline_schedule_id | Attribute | Type | required | Description | |--------------|---------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | ```shell curl --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" @@ -150,7 +150,7 @@ PUT /projects/:id/pipeline_schedules/:pipeline_schedule_id | Attribute | Type | required | Description | |---------------|---------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | | `description` | string | no | The description of pipeline schedule | | `ref` | string | no | The branch/tag name will be triggered | | `cron` | string | no | The cron (e.g. `0 1 * * *`) ([Cron syntax](https://en.wikipedia.org/wiki/Cron)) | @@ -200,7 +200,7 @@ POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/take_ownership | Attribute | Type | required | Description | |---------------|---------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | ```shell curl --request POST --header "PRIVATE-TOKEN: hf2CvZXB9w8Uc5pZKpSB" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/take_ownership" @@ -245,7 +245,7 @@ DELETE /projects/:id/pipeline_schedules/:pipeline_schedule_id | Attribute | Type | required | Description | |----------------|---------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | ```shell curl --request DELETE --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" @@ -286,18 +286,18 @@ curl --request DELETE --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gi Trigger a new scheduled pipeline, which runs immediately. The next scheduled run of this pipeline is not affected. -```text +```plaintext POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/play ``` | Attribute | Type | required | Description | | ---------------- | --------- | ---------- | -------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | Example request: -```sh +```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/pipeline_schedules/1/play ``` @@ -324,7 +324,7 @@ POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables | Attribute | Type | required | Description | |------------------------|----------------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | | `key` | string | yes | The `key` of a variable; must have no more than 255 characters; only `A-Z`, `a-z`, `0-9`, and `_` are allowed | | `value` | string | yes | The `value` of a variable | | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file` | @@ -352,7 +352,7 @@ PUT /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables/:key | Attribute | Type | required | Description | |------------------------|----------------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | | `key` | string | yes | The `key` of a variable | | `value` | string | yes | The `value` of a variable | | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file` | @@ -380,7 +380,7 @@ DELETE /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables/:key | Attribute | Type | required | Description | |------------------------|----------------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `pipeline_schedule_id` | integer | yes | The pipeline schedule id | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | | `key` | string | yes | The `key` of a variable | ```shell diff --git a/doc/api/pipeline_triggers.md b/doc/api/pipeline_triggers.md index 55c6e37c164..1a63a04be71 100644 --- a/doc/api/pipeline_triggers.md +++ b/doc/api/pipeline_triggers.md @@ -43,7 +43,7 @@ GET /projects/:id/triggers/:trigger_id | Attribute | Type | required | Description | |--------------|---------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `trigger_id` | integer | yes | The trigger id | +| `trigger_id` | integer | yes | The trigger ID | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/triggers/5" @@ -101,7 +101,7 @@ PUT /projects/:id/triggers/:trigger_id | Attribute | Type | required | Description | |---------------|---------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `trigger_id` | integer | yes | The trigger id | +| `trigger_id` | integer | yes | The trigger ID | | `description` | string | no | The trigger name | ```shell @@ -131,7 +131,7 @@ DELETE /projects/:id/triggers/:trigger_id | Attribute | Type | required | Description | |----------------|---------|----------|--------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `trigger_id` | integer | yes | The trigger id | +| `trigger_id` | integer | yes | The trigger ID | ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/triggers/5" diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 5b67df14ace..e84d7663bdb 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -1,5 +1,12 @@ # Pipelines API +## Pipelines pagination + +By default, `GET` requests return 20 results at a time because the API results +are paginated. + +Read more on [pagination](README.md#pagination). + ## List project pipelines > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 @@ -130,6 +137,62 @@ Example of response ] ``` +### Get a pipeline's test report + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202525) in GitLab 13.0. + +CAUTION: **Caution:** +This API route is part of the [JUnit test report](../ci/junit_test_reports.md) feature. It is protected by a [feature flag](../development/feature_flags/index.md) that is **disabled** due to performance issues with very large data sets. See [the documentation for the feature](../ci/junit_test_reports.md#enabling-the-feature) for further details. + +```plaintext +GET /projects/:id/pipelines/:pipeline_id/test_report +``` + +| Attribute | Type | Required | Description | +|------------|---------|----------|---------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `pipeline_id` | integer | yes | The ID of a pipeline | + +Sample request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipelines/46/test_report" +``` + +Sample response: + +```json +{ + "total_time": 5, + "total_count": 1, + "success_count": 1, + "failed_count": 0, + "skipped_count": 0, + "error_count": 0, + "test_suites": [ + { + "name": "Secure", + "total_time": 5, + "total_count": 1, + "success_count": 1, + "failed_count": 0, + "skipped_count": 0, + "error_count": 0, + "test_cases": [ + { + "status": "success", + "name": "Security Reports can create an auto-remediation MR", + "classname": "vulnerability_management_spec", + "execution_time": 5, + "system_output": null, + "stack_trace": null + } + ] + } + ] +} +``` + ## Create a new pipeline > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7209) in GitLab 8.14 @@ -225,7 +288,7 @@ Response: } ``` -## Cancel a pipelines jobs +## Cancel a pipeline's jobs > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 diff --git a/doc/api/project_badges.md b/doc/api/project_badges.md index 4afb898eb91..4adeb50deca 100644 --- a/doc/api/project_badges.md +++ b/doc/api/project_badges.md @@ -7,7 +7,7 @@ Badges support placeholders that will be replaced in real time in both the link and image URL. The allowed placeholders are: - **%{project_path}**: will be replaced by the project path. -- **%{project_id}**: will be replaced by the project id. +- **%{project_id}**: will be replaced by the project ID. - **%{default_branch}**: will be replaced by the project default branch. - **%{commit_sha}**: will be replaced by the last project's commit sha. diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 21a90670aa6..ae2fbcec0ff 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -18,7 +18,7 @@ data file uploads to the final server. From GitLab 10.7, the `upload[url]` parameter is required if the `upload` parameter is present. -```text +```plaintext POST /projects/:id/export ``` @@ -42,11 +42,14 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab } ``` +NOTE: **Note:** +The upload request will be sent with `Content-Type: application/gzip` header. Ensure that your pre-signed URL includes this as part of the signature. + ## Export status Get the status of export. -```text +```plaintext GET /projects/:id/export ``` @@ -99,7 +102,7 @@ an email notifying the user to download the file, uploading the exported file to Download the finished export. -```text +```plaintext GET /projects/:id/export/download ``` @@ -118,7 +121,7 @@ ls *export.tar.gz ## Import a file -```text +```plaintext POST /projects/import ``` @@ -182,7 +185,7 @@ requests.post(url, headers=headers, data=data, files=files) Get the status of an import. -```text +```plaintext GET /projects/:id/import ``` diff --git a/doc/api/project_repository_storage_moves.md b/doc/api/project_repository_storage_moves.md new file mode 100644 index 00000000000..8df472f193f --- /dev/null +++ b/doc/api/project_repository_storage_moves.md @@ -0,0 +1,80 @@ +# Project repository storage move API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31285) in GitLab 13.0. + +Project repository storage can be moved. To retrieve project repository storage moves using the API, you must [authenticate yourself](README.md#authentication) as an administrator. + +## Retrieve all project repository storage moves + +```plaintext +GET /project_repository_storage_moves +``` + +By default, `GET` requests return 20 results at a time because the API results +are [paginated](README.md#pagination). + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://primary.example.com/api/v4/project_repository_storage_moves' +``` + +Example response: + +```json +[ + { + "id": 1, + "created_at": "2020-05-07T04:27:17.234Z", + "state": "scheduled", + "source_storage_name": "default", + "destination_storage_name": "storage2", + "project": { + "id": 1, + "description": null, + "name": "project1", + "name_with_namespace": "John Doe2 / project1", + "path": "project1", + "path_with_namespace": "namespace1/project1", + "created_at": "2020-05-07T04:27:17.016Z" + } +] +``` + +## Get a single project repository storage move + +```plaintext +GET /project_repository_storage_moves/:id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer | yes | The ID of the project repository storage move | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://primary.example.com/api/v4/project_repository_storage_moves/1' +``` + +Example response: + +```json +{ + "id": 1, + "created_at": "2020-05-07T04:27:17.234Z", + "state": "scheduled", + "source_storage_name": "default", + "destination_storage_name": "storage2", + "project": { + "id": 1, + "description": null, + "name": "project1", + "name_with_namespace": "John Doe2 / project1", + "path": "project1", + "path_with_namespace": "namespace1/project1", + "created_at": "2020-05-07T04:27:17.016Z" +} +``` diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index 9a37c675615..e435f87dcdb 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -60,7 +60,9 @@ Parameters: }, "updated_at": "2012-06-28T10:52:04Z", "created_at": "2012-06-28T10:52:04Z", - "web_url": "http://example.com/example/example/snippets/1" + "project_id": 1, + "web_url": "http://example.com/example/example/snippets/1", + "raw_url": "http://example.com/example/example/snippets/1/raw" } ``` diff --git a/doc/api/projects.md b/doc/api/projects.md index 16c8569349c..8cb8961bafa 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -9,10 +9,8 @@ Values for the project visibility level are: - `private`: Project access must be granted explicitly for each user. - - `internal`: The project can be cloned by any logged in user. - - `public`: The project can be accessed without any authentication. @@ -22,11 +20,9 @@ There are currently three options for `merge_method` to choose from: - `merge`: A merge commit is created for every merge, and merging is allowed as long as there are no conflicts. - - `rebase_merge`: A merge commit is created for every merge, but merging is only allowed if fast-forward merge is possible. This way you could make sure that if this merge request would build, after merging to target branch it would also build. - - `ff`: No merge commits are created and all merges are fast-forwarded, which means that merging is only allowed if the branch could be fast-forwarded. @@ -162,7 +158,7 @@ When the user is authenticated and `simple` is not set this returns something li "merge_method": "merge", "autoclose_referenced_issues": true, "suggestion_commit_message": null, - "marked_for_deletion_at": "2020-04-03", // to be deprecated in GitLab 13.0 in favor of marked_for_deletion_on + "marked_for_deletion_at": "2020-04-03", // Deprecated and will be removed in API v5 in favor of marked_for_deletion_on "marked_for_deletion_on": "2020-04-03", "statistics": { "commit_count": 37, @@ -288,7 +284,7 @@ When the user is authenticated and `simple` is not set this returns something li ``` NOTE: **Note:** -For users on GitLab [Silver, Premium, or higher](https://about.gitlab.com/pricing/) the `marked_for_deletion_at` attribute will be deprecated in GitLab 13.0 in favor of the `marked_for_deletion_on` attribute. +For users on GitLab [Silver, Premium, or higher](https://about.gitlab.com/pricing/) the `marked_for_deletion_at` attribute has been deprecated and will be removed in API v5 in favor of the `marked_for_deletion_on` attribute. Users on GitLab [Starter, Bronze, or higher](https://about.gitlab.com/pricing/) will also see the `approvals_before_merge` parameter: @@ -411,7 +407,7 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "merge_method": "merge", "autoclose_referenced_issues": true, "suggestion_commit_message": null, - "marked_for_deletion_at": "2020-04-03", // to be deprecated in GitLab 13.0 in favor of marked_for_deletion_on + "marked_for_deletion_at": "2020-04-03", // Deprecated and will be removed in API v5 in favor of marked_for_deletion_on "marked_for_deletion_on": "2020-04-03", "statistics": { "commit_count": 37, @@ -798,7 +794,9 @@ GET /projects/:id "enabled": false, "keep_n": null, "older_than": null, - "name_regex": null, + "name_regex": null, // to be deprecated in GitLab 13.0 in favor of `name_regex_delete` + "name_regex_delete": null, + "name_regex_keep": null, "next_run_at": "2020-01-07T21:42:58.658Z" }, "created_at": "2013-09-30T13:46:02Z", @@ -877,7 +875,7 @@ GET /projects/:id "service_desk_address": null, "autoclose_referenced_issues": true, "suggestion_commit_message": null, - "marked_for_deletion_at": "2020-04-03", // to be deprecated in GitLab 13.0 in favor of marked_for_deletion_on + "marked_for_deletion_at": "2020-04-03", // Deprecated and will be removed in API v5 in favor of marked_for_deletion_on "marked_for_deletion_on": "2020-04-03", "statistics": { "commit_count": 37, @@ -912,7 +910,7 @@ the `approvals_before_merge` parameter: } ``` -**Note**: The `web_url` and `avatar_url` attributes on `namespace` were [introduced][ce-27427] in GitLab 11.11. +**Note**: The `web_url` and `avatar_url` attributes on `namespace` were [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/27427) in GitLab 11.11. If the project is a fork, and you provide a valid token to authenticate, the `forked_from_project` field will appear in the response. @@ -1031,9 +1029,10 @@ POST /projects | `snippets_access_level` | string | no | One of `disabled`, `private` or `enabled` | | `pages_access_level` | string | no | One of `disabled`, `private`, `enabled` or `public` | | `emails_disabled` | boolean | no | Disable email notifications | +| `show_default_award_emojis` | boolean | no | Show default award emojis | | `resolve_outdated_diff_discussions` | boolean | no | Automatically resolve merge request diffs discussions on lines changed with a push | | `container_registry_enabled` | boolean | no | Enable container registry for this project | -| `container_expiration_policy_attributes` | hash | no | Update the container expiration policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `enabled` (boolean) | +| `container_expiration_policy_attributes` | hash | no | Update the image expiration policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean) | | `shared_runners_enabled` | boolean | no | Enable shared runners for this project | | `visibility` | string | no | See [project visibility level](#project-visibility-level) | | `import_url` | string | no | URL to import repository from | @@ -1100,6 +1099,7 @@ POST /projects/user/:user_id | `snippets_access_level` | string | no | One of `disabled`, `private` or `enabled` | | `pages_access_level` | string | no | One of `disabled`, `private`, `enabled` or `public` | | `emails_disabled` | boolean | no | Disable email notifications | +| `show_default_award_emojis` | boolean | no | Show default award emojis | | `resolve_outdated_diff_discussions` | boolean | no | Automatically resolve merge request diffs discussions on lines changed with a push | | `container_registry_enabled` | boolean | no | Enable container registry for this project | | `shared_runners_enabled` | boolean | no | Enable shared runners for this project | @@ -1168,9 +1168,10 @@ PUT /projects/:id | `snippets_access_level` | string | no | One of `disabled`, `private` or `enabled` | | `pages_access_level` | string | no | One of `disabled`, `private`, `enabled` or `public` | | `emails_disabled` | boolean | no | Disable email notifications | +| `show_default_award_emojis` | boolean | no | Show default award emojis | | `resolve_outdated_diff_discussions` | boolean | no | Automatically resolve merge request diffs discussions on lines changed with a push | | `container_registry_enabled` | boolean | no | Enable container registry for this project | -| `container_expiration_policy_attributes` | hash | no | Update the container expiration policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `enabled` (boolean) | +| `container_expiration_policy_attributes` | hash | no | Update the image expiration policy for this project. Accepts: `cadence` (string), `keep_n` (string), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean) | | `shared_runners_enabled` | boolean | no | Enable shared runners for this project | | `visibility` | string | no | See [project visibility level](#project-visibility-level) | | `import_url` | string | no | URL to import repository from | @@ -2252,5 +2253,3 @@ GET /projects/:id/snapshot | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `wiki` | boolean | no | Whether to download the wiki, rather than project, repository | - -[ce-27427]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/27427 diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 47cef0e5fa0..9c3cac3c64f 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -96,7 +96,6 @@ Example response: ], "commit_path":"/root/awesome-app/commit/588440f66559714280628a4f9799f0c4eb880a4a", "tag_path":"/root/awesome-app/-/tags/v0.11.1", - "evidence_sha":"760d6cdfb0879c3ffedec13af470e0f71cf52c6cde4d", "assets":{ "count":6, "sources":[ @@ -133,6 +132,13 @@ Example response: ], "evidence_file_path":"https://gitlab.example.com/root/awesome-app/-/releases/v0.2/evidence.json" }, + "evidences":[ + { + sha: "760d6cdfb0879c3ffedec13af470e0f71cf52c6cde4d", + filepath: "https://gitlab.example.com/root/awesome-app/-/releases/v0.2/evidence.json", + collected_at: "2019-01-03T01:56:19.539Z" + } + ] }, { "tag_name":"v0.1", @@ -165,7 +171,6 @@ Example response: "committer_email":"admin@example.com", "committed_date":"2019-01-03T01:53:28.000Z" }, - "evidence_sha":"760d6cdfb0879c3ffedec13af470e0f71cf52c6cde4d", "assets":{ "count":4, "sources":[ @@ -191,6 +196,13 @@ Example response: ], "evidence_file_path":"https://gitlab.example.com/root/awesome-app/-/releases/v0.1/evidence.json" }, + "evidences":[ + { + sha: "c3ffedec13af470e760d6cdfb08790f71cf52c6cde4d", + filepath: "https://gitlab.example.com/root/awesome-app/-/releases/v0.1/evidence.json", + collected_at: "2019-01-03T01:55:18.203Z" + } + ] } ] ``` @@ -286,7 +298,6 @@ Example response: ], "commit_path":"/root/awesome-app/commit/588440f66559714280628a4f9799f0c4eb880a4a", "tag_path":"/root/awesome-app/-/tags/v0.11.1", - "evidence_sha":"760d6cdfb0879c3ffedec13af470e0f71cf52c6cde4d", "assets":{ "count":5, "sources":[ @@ -314,9 +325,15 @@ Example response: "url":"https://gitlab.example.com/root/awesome-app/-/tags/v0.11.1/binaries/linux-amd64", "external":true } - ], - "evidence_url":"https://gitlab.example.com/root/awesome-app/-/releases/v0.1/evidence.json" + ] }, + "evidences":[ + { + sha: "760d6cdfb0879c3ffedec13af470e0f71cf52c6cde4d", + filepath: "https://gitlab.example.com/root/awesome-app/-/releases/v0.1/evidence.json", + collected_at: "2019-07-16T14:00:12.256Z" + } + ] } ``` @@ -338,7 +355,7 @@ POST /projects/:id/releases | `milestones` | array of string | no | The title of each milestone the release is associated with. | | `assets:links` | array of hash | no | An array of assets links. | | `assets:links:name`| string | required by: `assets:links` | The name of the link. | -| `assets:links:url` | string | required by: `assets:links` | The url of the link. | +| `assets:links:url` | string | required by: `assets:links` | The URL of the link. | | `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases.md). | `released_at` | datetime | no | The date when the release will be/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index bf882ef35c0..f380ba5a1b2 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -21,7 +21,7 @@ GET /projects/:id/releases/:tag_name/assets/links Example request: ```shell -curl --header "PRIVATE-TOKEN: gDybLx3yrUK_HLp3qPjS" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links" +curl --header "PRIVATE-TOKEN: n671WNGecHugsdEDPsyo" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links" ``` Example response: @@ -55,12 +55,12 @@ GET /projects/:id/releases/:tag_name/assets/links/:link_id | ------------- | -------------- | -------- | --------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../README.md#namespaced-path-encoding). | | `tag_name` | string | yes | The tag associated with the Release. | -| `link_id` | integer | yes | The id of the link. | +| `link_id` | integer | yes | The ID of the link. | Example request: ```shell -curl --header "PRIVATE-TOKEN: gDybLx3yrUK_HLp3qPjS" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" +curl --header "PRIVATE-TOKEN: n671WNGecHugsdEDPsyo" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" ``` Example response: @@ -93,7 +93,7 @@ Example request: ```shell curl --request POST \ - --header "PRIVATE-TOKEN: gDybLx3yrUK_HLp3qPjS" \ + --header "PRIVATE-TOKEN: n671WNGecHugsdEDPsyo" \ --data name="awesome-v0.2.dmg" \ --data url="http://192.168.10.15:3000" \ "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links" @@ -122,7 +122,7 @@ PUT /projects/:id/releases/:tag_name/assets/links/:link_id | ------------- | -------------- | -------- | --------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../README.md#namespaced-path-encoding). | | `tag_name` | string | yes | The tag associated with the Release. | -| `link_id` | integer | yes | The id of the link. | +| `link_id` | integer | yes | The ID of the link. | | `name` | string | no | The name of the link. | | `url` | string | no | The URL of the link. | @@ -132,7 +132,7 @@ You have to specify at least one of `name` or `url` Example request: ```shell -curl --request PUT --data name="new name" --header "PRIVATE-TOKEN: gDybLx3yrUK_HLp3qPjS" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" +curl --request PUT --data name="new name" --header "PRIVATE-TOKEN: n671WNGecHugsdEDPsyo" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" ``` Example response: @@ -158,12 +158,12 @@ DELETE /projects/:id/releases/:tag_name/assets/links/:link_id | ------------- | -------------- | -------- | --------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../README.md#namespaced-path-encoding). | | `tag_name` | string | yes | The tag associated with the Release. | -| `link_id` | integer | yes | The id of the link. | +| `link_id` | integer | yes | The ID of the link. | Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: gDybLx3yrUK_HLp3qPjS" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" +curl --request DELETE --header "PRIVATE-TOKEN: n671WNGecHugsdEDPsyo" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" ``` Example response: diff --git a/doc/api/remote_mirrors.md b/doc/api/remote_mirrors.md index 0ffff194976..e46a890cbd4 100644 --- a/doc/api/remote_mirrors.md +++ b/doc/api/remote_mirrors.md @@ -11,13 +11,13 @@ outlined below. Returns an Array of remote mirrors and their statuses: -```text +```plaintext GET /projects/:id/remote_mirrors ``` Example request: -```sh +```shell curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors' ``` @@ -33,6 +33,7 @@ Example response: "last_update_at": "2020-01-06T17:32:02.823Z", "last_update_started_at": "2020-01-06T17:31:55.864Z", "only_protected_branches": true, + "keep_divergent_refs": true, "update_status": "finished", "url": "https://*****:*****@gitlab.com/gitlab-org/security/gitlab.git" } @@ -49,7 +50,7 @@ and password information. Create a remote mirror for a project. The mirror will be disabled by default. You can enable it by including the optional parameter `enabled` when creating it: -```text +```plaintext POST /projects/:id/remote_mirrors ``` @@ -58,10 +59,11 @@ POST /projects/:id/remote_mirrors | `url` | String | yes | The URL of the remote repository to be mirrored. | | `enabled` | Boolean | no | Determines if the mirror is enabled. | | `only_protected_branches` | Boolean | no | Determines if only protected branches are mirrored. | +| `keep_divergent_refs` | Boolean | no | Determines if divergent refs are skipped. | Example request: -```sh +```shell curl --request POST --data "url=https://username:token@example.com/gitlab/example.git" --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors' ``` @@ -76,6 +78,7 @@ Example response: "last_update_at": null, "last_update_started_at": null, "only_protected_branches": false, + "keep_divergent_refs": false, "update_status": "none", "url": "https://*****:*****@example.com/gitlab/example.git" } @@ -88,7 +91,7 @@ Example response: Toggle a remote mirror on or off, or change which types of branches are mirrored: -```text +```plaintext PUT /projects/:id/remote_mirrors/:mirror_id ``` @@ -97,10 +100,11 @@ PUT /projects/:id/remote_mirrors/:mirror_id | `mirror_id` | Integer | yes | The remote mirror ID. | | `enabled` | Boolean | no | Determines if the mirror is enabled. | | `only_protected_branches` | Boolean | no | Determines if only protected branches are mirrored. | +| `keep_divergent_refs` | Boolean | no | Determines if divergent refs are skipped. | Example request: -```sh +```shell curl --request PUT --data "enabled=false" --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors/101486' ``` @@ -115,6 +119,7 @@ Example response: "last_update_at": "2020-01-06T17:32:02.823Z", "last_update_started_at": "2020-01-06T17:31:55.864Z", "only_protected_branches": true, + "keep_divergent_refs": true, "update_status": "finished", "url": "https://*****:*****@gitlab.com/gitlab-org/security/gitlab.git" } diff --git a/doc/api/repositories.md b/doc/api/repositories.md index f261c9ab9f7..440db06792c 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -107,6 +107,8 @@ Parameters: Get an archive of the repository. This endpoint can be accessed without authentication if the repository is publicly accessible. +This endpoint has a rate limit threshold of 5 requests per minute. + ```plaintext GET /projects/:id/repository/archive[.format] ``` diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index e6425c3fe17..a8b58d90c34 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -47,7 +47,7 @@ Example response: Parameters: -- `file_path` (required) - Url encoded full path to new file. Ex. lib%2Fclass%2Erb +- `file_path` (required) - URL encoded full path to new file. Ex. lib%2Fclass%2Erb - `ref` (required) - The name of branch, tag or commit NOTE: **Note:** @@ -65,7 +65,7 @@ curl --head --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.exampl Example response: -```text +```plaintext HTTP/1.1 200 OK ... X-Gitlab-Blob-Id: 79f7bbd25901e8334750839545a9bd021f0e4c83 @@ -122,7 +122,7 @@ Example response: Parameters: -- `file_path` (required) - Url encoded full path to new file. Ex. lib%2Fclass%2Erb +- `file_path` (required) - URL encoded full path to new file. Ex. lib%2Fclass%2Erb - `ref` (required) - The name of branch, tag or commit NOTE: **Note:** @@ -134,7 +134,7 @@ curl --head --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.exampl Example response: -```text +```plaintext HTTP/1.1 200 OK ... X-Gitlab-Blob-Id: 79f7bbd25901e8334750839545a9bd021f0e4c83 @@ -161,7 +161,7 @@ curl --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.example.com/a Parameters: -- `file_path` (required) - Url encoded full path to new file. Ex. lib%2Fclass%2Erb +- `file_path` (required) - URL encoded full path to new file. Ex. lib%2Fclass%2Erb - `ref` (required) - The name of branch, tag or commit NOTE: **Note:** @@ -193,7 +193,7 @@ Example response: Parameters: -- `file_path` (required) - Url encoded full path to new file. Ex. lib%2Fclass%2Erb +- `file_path` (required) - URL encoded full path to new file. Ex. lib%2Fclass%2Erb - `branch` (required) - Name of the branch - `start_branch` (optional) - Name of the branch to start the new commit from - `encoding` (optional) - Change encoding to 'base64'. Default is text. @@ -228,7 +228,7 @@ Example response: Parameters: -- `file_path` (required) - Url encoded full path to new file. Ex. lib%2Fclass%2Erb +- `file_path` (required) - URL encoded full path to new file. Ex. lib%2Fclass%2Erb - `branch` (required) - Name of the branch - `start_branch` (optional) - Name of the branch to start the new commit from - `encoding` (optional) - Change encoding to 'base64'. Default is text. @@ -236,7 +236,7 @@ Parameters: - `author_name` (optional) - Specify the commit author's name - `content` (required) - New file content - `commit_message` (required) - Commit message -- `last_commit_id` (optional) - Last known file commit id +- `last_commit_id` (optional) - Last known file commit ID If the commit fails for any reason we return a 400 error with a non-specific error message. Possible causes for a failed commit include: @@ -265,10 +265,10 @@ curl --request DELETE --header 'PRIVATE-TOKEN: <your_access_token>' --header "Co Parameters: -- `file_path` (required) - Url encoded full path to new file. Ex. lib%2Fclass%2Erb +- `file_path` (required) - URL encoded full path to new file. Ex. lib%2Fclass%2Erb - `branch` (required) - Name of the branch - `start_branch` (optional) - Name of the branch to start the new commit from - `author_email` (optional) - Specify the commit author's email address - `author_name` (optional) - Specify the commit author's name - `commit_message` (required) - Commit message -- `last_commit_id` (optional) - Last known file commit id +- `last_commit_id` (optional) - Last known file commit ID diff --git a/doc/api/runners.md b/doc/api/runners.md index 21d768a1605..5db1f116f6c 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -162,9 +162,9 @@ GET /runners/:id curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/runners/6" ``` -CAUTION: **Deprecation** -The `token` attribute in the response is deprecated [since GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/214320). -It will be removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/214322). +NOTE: **Note:** +The `token` attribute in the response was deprecated [in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/214320). +and removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/214322). Example response: @@ -190,7 +190,6 @@ Example response: "path_with_namespace": "gitlab-org/gitlab-foss" } ], - "token": "205086a8e3b9a2b818ffac9b89d102", "revision": null, "tag_list": [ "ruby", @@ -225,9 +224,9 @@ PUT /runners/:id curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/runners/6" --form "description=test-1-20150125-test" --form "tag_list=ruby,mysql,tag1,tag2" ``` -CAUTION: **Deprecation** -The `token` attribute in the response is deprecated [since GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/214320). -It will be removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/214322). +NOTE: **Note:** +The `token` attribute in the response was deprecated [in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/214320). +and removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/214322). Example response: @@ -253,7 +252,6 @@ Example response: "path_with_namespace": "gitlab-org/gitlab-foss" } ], - "token": "205086a8e3b9a2b818ffac9b89d102", "revision": null, "tag_list": [ "ruby", diff --git a/doc/api/scim.md b/doc/api/scim.md index eaa56b0d0dd..4300c9efa3d 100644 --- a/doc/api/scim.md +++ b/doc/api/scim.md @@ -4,6 +4,9 @@ The SCIM API implements the [the RFC7644 protocol](https://tools.ietf.org/html/rfc7644). +CAUTION: **Caution:** +This API is for internal system use for connecting with a SCIM provider. While it can be used directly, it is subject to change without notice. + NOTE: **Note:** [Group SSO](../user/group/saml_sso/index.md) must be enabled for the group. For more information, see [SCIM setup documentation](../user/group/saml_sso/scim_setup.md#requirements). @@ -13,7 +16,7 @@ NOTE: **Note:** This endpoint is used as part of the SCIM syncing mechanism and it only returns a single user based on a unique ID which should match the `extern_uid` of the user. -```text +```plaintext GET /api/scim/v2/groups/:group_path/Users ``` @@ -69,7 +72,7 @@ Example response: ## Get a single SAML user -```text +```plaintext GET /api/scim/v2/groups/:group_path/Users/:id ``` @@ -110,7 +113,7 @@ Example response: ## Create a SAML user -```text +```plaintext POST /api/scim/v2/groups/:group_path/Users/ ``` @@ -158,15 +161,15 @@ Returns a `201` status code if successful. Fields that can be updated are: -| SCIM/IdP field | GitLab field | -|:----------|:--------| -| id/externalId | extern_uid | -| name.formatted | name | -| emails\[type eq "work"\].value | email | -| active | Identity removal if `active = false` | -| userName | username | +| SCIM/IdP field | GitLab field | +|:---------------------------------|:---------------------------------------| +| `id/externalId` | `extern_uid` | +| `name.formatted` | `name` | +| `emails\[type eq "work"\].value` | `email` | +| `active` | Identity removal if `active` = `false` | +| `userName` | `username` | -```text +```plaintext PATCH /api/scim/v2/groups/:group_path/Users/:id ``` @@ -190,7 +193,7 @@ Returns an empty response with a `204` status code if successful. Removes the user's SSO identity and group membership. -```text +```plaintext DELETE /api/scim/v2/groups/:group_path/Users/:id ``` diff --git a/doc/api/search.md b/doc/api/search.md index e1c70fe56a7..7940a2fa4e3 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -17,7 +17,7 @@ GET /search | `scope` | string | yes | The scope to search in | | `search` | string | yes | The search query | -Search the expression within the specified scope. Currently these scopes are supported: projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs, users. +Search the expression within the specified scope. Currently these scopes are supported: projects, issues, merge_requests, milestones, snippet_titles, users. If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs and commits. Find more about [the feature](../integration/elasticsearch.md). **(STARTER)** @@ -253,39 +253,6 @@ Example response: ] ``` -### Scope: snippet_blobs - -This scope will be disabled after GitLab 13.0. - -```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=snippet_blobs&search=test -``` - -Example response: - -```json -[ - { - "id": 50, - "title": "Sample file", - "file_name": "file.rb", - "description": "Simple ruby file", - "author": { - "id": 1, - "name": "Administrator", - "username": "root", - "state": "active", - "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", - "web_url": "http://localhost:3000/root" - }, - "updated_at": "2018-02-06T12:49:29.104Z", - "created_at": "2017-11-28T08:20:18.071Z", - "project_id": 9, - "web_url": "http://localhost:3000/root/jira-test/snippets/50" - } -] -``` - ### Scope: wiki_blobs **(STARTER)** This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. diff --git a/doc/api/services.md b/doc/api/services.md index 3a5268f4271..d435dffa651 100644 --- a/doc/api/services.md +++ b/doc/api/services.md @@ -189,9 +189,9 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `new_issue_url` | string | true | New Issue url | -| `issues_url` | string | true | Issue url | -| `project_url` | string | true | Project url | +| `new_issue_url` | string | true | New Issue URL | +| `issues_url` | string | true | Issue URL | +| `project_url` | string | true | Project URL | | `description` | string | false | Description | | `title` | string | false | Title | | `push_events` | boolean | false | Enable notifications for push events | @@ -331,6 +331,51 @@ Get Unify Circuit service settings for a project. GET /projects/:id/services/unify-circuit ``` +## Webex Teams + +Webex Teams collaboration tool. + +### Create/Edit Webex Teams service + +Set Webex Teams service for a project. + +```plaintext +PUT /projects/:id/services/webex-teams +``` + +Parameters: + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `webhook` | string | true | The Webex Teams webhook. For example, `https://api.ciscospark.com/v1/webhooks/incoming/...`. | +| `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | +| `branches_to_be_notified` | string | all | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected" | +| `push_events` | boolean | false | Enable notifications for push events | +| `issues_events` | boolean | false | Enable notifications for issue events | +| `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | +| `merge_requests_events` | boolean | false | Enable notifications for merge request events | +| `tag_push_events` | boolean | false | Enable notifications for tag push events | +| `note_events` | boolean | false | Enable notifications for note events | +| `confidential_note_events` | boolean | false | Enable notifications for confidential note events | +| `pipeline_events` | boolean | false | Enable notifications for pipeline events | +| `wiki_page_events` | boolean | false | Enable notifications for wiki page events | + +### Delete Webex Teams service + +Delete Webex Teams service for a project. + +```plaintext +DELETE /projects/:id/services/webex-teams +``` + +### Get Webex Teams service settings + +Get Webex Teams service settings for a project. + +```plaintext +GET /projects/:id/services/webex-teams +``` + ## Custom Issue Tracker Custom issue tracker @@ -347,11 +392,11 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `new_issue_url` | string | true | New Issue url -| `issues_url` | string | true | Issue url -| `project_url` | string | true | Project url -| `description` | string | false | Description -| `title` | string | false | Title +| `new_issue_url` | string | true | New Issue URL | +| `issues_url` | string | true | Issue URL | +| `project_url` | string | true | Project URL | +| `description` | string | false | Description | +| `title` | string | false | Title | | `push_events` | boolean | false | Enable notifications for push events | ### Delete Custom Issue Tracker service @@ -567,7 +612,7 @@ Set Hangouts Chat service for a project. PUT /projects/:id/services/hangouts-chat ``` ->**Note:** Specific event parameters (for example, `push_events` flag) were [introduced in v10.4][11435] +>**Note:** Specific event parameters (for example, `push_events` flag) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) Parameters: @@ -1034,9 +1079,9 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `new_issue_url` | string | true | New Issue url | -| `project_url` | string | true | Project url | -| `issues_url` | string | true | Issue url | +| `new_issue_url` | string | true | New Issue URL | +| `project_url` | string | true | Project URL | +| `issues_url` | string | true | Issue URL | | `description` | string | false | Description | | `push_events` | boolean | false | Enable notifications for push events | @@ -1068,7 +1113,7 @@ Set Slack service for a project. PUT /projects/:id/services/slack ``` ->**Note:** Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4][11435] +>**Note:** Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) Parameters: @@ -1177,7 +1222,7 @@ Set Mattermost service for a project. PUT /projects/:id/services/mattermost ``` ->**Note:** Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4][11435] +>**Note:** Specific event parameters (for example, `push_events` flag and `push_channel`) were [introduced in v10.4](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435) Parameters: @@ -1303,6 +1348,9 @@ GET /projects/:id/services/jenkins A continuous integration and build server +NOTE: **Note:** +This service was [removed in v13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/1600) + ### Create/Edit Jenkins CI (Deprecated) service Set Jenkins CI (Deprecated) service for a project. @@ -1369,8 +1417,6 @@ Get MockCI service settings for a project. GET /projects/:id/services/mock-ci ``` -[11435]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11435 - ## YouTrack YouTrack issue tracker @@ -1387,8 +1433,8 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `issues_url` | string | true | Issue url | -| `project_url` | string | true | Project url | +| `issues_url` | string | true | Issue URL | +| `project_url` | string | true | Project URL | | `description` | string | false | Description | | `push_events` | boolean | false | Enable notifications for push events | diff --git a/doc/api/settings.md b/doc/api/settings.md index cf48048c830..f63d126742a 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -69,7 +69,9 @@ Example response: "asset_proxy_enabled": true, "asset_proxy_url": "https://assets.example.com", "asset_proxy_whitelist": ["example.com", "*.example.com", "your-instance.com"], - "npm_package_requests_forwarding": true + "npm_package_requests_forwarding": true, + "issues_create_limit": 300, + "raw_blob_request_limit": 300 } ``` @@ -156,7 +158,9 @@ Example response: "allow_local_requests_from_hooks_and_services": true, "allow_local_requests_from_web_hooks_and_services": true, "allow_local_requests_from_system_hooks": false, - "npm_package_requests_forwarding": true + "npm_package_requests_forwarding": true, + "issues_create_limit": 300, + "raw_blob_request_limit": 300 } ``` @@ -228,12 +232,15 @@ are listed in the descriptions of the relevant settings. | `elasticsearch_aws` | boolean | no | **(PREMIUM)** Enable the use of AWS hosted Elasticsearch | | `elasticsearch_aws_region` | string | no | **(PREMIUM)** The AWS region the Elasticsearch domain is configured | | `elasticsearch_aws_secret_access_key` | string | no | **(PREMIUM)** AWS IAM secret access key | +| `elasticsearch_indexed_field_length_limit` | integer | no | **(PREMIUM)** Maximum size of text fields that will be indexed by Elasticsearch. 0 value means no limit. This does not apply to repository and wiki indexing. | | `elasticsearch_indexing` | boolean | no | **(PREMIUM)** Enable Elasticsearch indexing | | `elasticsearch_limit_indexing` | boolean | no | **(PREMIUM)** Limit Elasticsearch to index certain namespaces and projects | +| `elasticsearch_max_bulk_concurrency` | integer | no | **(PREMIUM)** Maximum concurrency of Elasticsearch bulk requests per indexing operation. This only applies to repository indexing operations. | +| `elasticsearch_max_bulk_size_mb` | integer | no | **(PREMIUM)** Maximum size of Elasticsearch bulk indexing requests in MB. This only applies to repository indexing operations. | | `elasticsearch_namespace_ids` | array of integers | no | **(PREMIUM)** The namespaces to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. | | `elasticsearch_project_ids` | array of integers | no | **(PREMIUM)** The projects to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. | | `elasticsearch_search` | boolean | no | **(PREMIUM)** Enable Elasticsearch search | -| `elasticsearch_url` | string | no | **(PREMIUM)** The url to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (for example, `http://localhost:9200, http://localhost:9201"`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (for example, `http://<username>:<password>@<elastic_host>:9200/`). | +| `elasticsearch_url` | string | no | **(PREMIUM)** The URL to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (for example, `http://localhost:9200, http://localhost:9201"`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (for example, `http://<username>:<password>@<elastic_host>:9200/`). | | `email_additional_text` | string | no | **(PREMIUM)** Additional text added to the bottom of every email for legal/auditing/compliance reasons | | `email_author_in_body` | boolean | no | Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead. | | `enabled_git_access_protocol` | string | no | Enabled protocols for Git access. Allowed values are: `ssh`, `http`, and `nil` to allow both protocols. | @@ -255,7 +262,7 @@ are listed in the descriptions of the relevant settings. | `grafana_enabled` | boolean | no | Enable Grafana. | | `grafana_url` | string | no | Grafana URL. | | `gravatar_enabled` | boolean | no | Enable Gravatar. | -| `hashed_storage_enabled` | boolean | no | Create new projects using hashed storage paths: Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance. (EXPERIMENTAL) | +| `hashed_storage_enabled` | boolean | no | Create new projects using hashed storage paths: Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance. (Always enabled since 13.0, configuration will be removed in 14.0) | | `help_page_hide_commercial_content` | boolean | no | Hide marketing-related entries from help. | | `help_page_support_url` | string | no | Alternate support URL for help page and help dropdown. | | `help_page_text` | string | no | Custom text displayed on the help page. | @@ -275,14 +282,7 @@ are listed in the descriptions of the relevant settings. | `max_attachment_size` | integer | no | Limit attachment size in MB | | `max_pages_size` | integer | no | Maximum size of pages repositories in MB | | `max_personal_access_token_lifetime` | integer | no | **(ULTIMATE ONLY)** Maximum allowable lifetime for personal access tokens in days | -| `metrics_enabled` | boolean | no | (**If enabled, requires:** `metrics_host`, `metrics_method_call_threshold`, `metrics_packet_size`, `metrics_pool_size`, `metrics_port`, `metrics_sample_interval` and `metrics_timeout`) Enable influxDB metrics. | -| `metrics_host` | string | required by: `metrics_enabled` | InfluxDB host. | -| `metrics_method_call_threshold` | integer | required by: `metrics_enabled` | A method call is only tracked when it takes longer than the given amount of milliseconds. | -| `metrics_packet_size` | integer | required by: `metrics_enabled` | The amount of data points to send in a single UDP packet. | -| `metrics_pool_size` | integer | required by: `metrics_enabled` | The amount of InfluxDB connections to keep open. | -| `metrics_port` | integer | required by: `metrics_enabled` | The UDP port to use for connecting to InfluxDB. | -| `metrics_sample_interval` | integer | required by: `metrics_enabled` | The sampling interval in seconds. | -| `metrics_timeout` | integer | required by: `metrics_enabled` | The amount of seconds after which InfluxDB will time out. | +| `metrics_method_call_threshold` | integer | no | A method call is only tracked when it takes longer than the given amount of milliseconds. | | `mirror_available` | boolean | no | Allow repository mirroring to configured by project Maintainers. If disabled, only Admins will be able to configure repository mirroring. | | `mirror_capacity_threshold` | integer | no | **(PREMIUM)** Minimum capacity to be available before scheduling more mirrors preemptively | | `mirror_max_capacity` | integer | no | **(PREMIUM)** Maximum number of mirrors that can be synchronizing at the same time. | @@ -324,13 +324,13 @@ are listed in the descriptions of the relevant settings. | `sign_in_text` | string | no | Text on the login page. | | `signup_enabled` | boolean | no | Enable registration. Default is `true`. | | `slack_app_enabled` | boolean | no | **(PREMIUM)** (**If enabled, requires:** `slack_app_id`, `slack_app_secret` and `slack_app_secret`) Enable Slack app. | -| `slack_app_id` | string | required by: `slack_app_enabled` | **(PREMIUM)** The app id of the Slack-app. | +| `slack_app_id` | string | required by: `slack_app_enabled` | **(PREMIUM)** The app ID of the Slack-app. | | `slack_app_secret` | string | required by: `slack_app_enabled` | **(PREMIUM)** The app secret of the Slack-app. | | `slack_app_verification_token` | string | required by: `slack_app_enabled` | **(PREMIUM)** The verification token of the Slack-app. | | `snowplow_collector_hostname` | string | required by: `snowplow_enabled` | The Snowplow collector hostname. (for example, `snowplow.trx.gitlab.net`) | | `snowplow_cookie_domain` | string | no | The Snowplow cookie domain. (for example, `.gitlab.com`) | | `snowplow_enabled` | boolean | no | Enable snowplow tracking. | -| `snowplow_app_id` | string | no | The Snowplow site name / application id. (for example, `gitlab`) | +| `snowplow_app_id` | string | no | The Snowplow site name / application ID. (for example, `gitlab`) | | `snowplow_iglu_registry_url` | string | no | The Snowplow base Iglu Schema Registry URL to use for custom context and self describing events'| | `sourcegraph_enabled` | boolean | no | Enables Sourcegraph integration. Default is `false`. **If enabled, requires** `sourcegraph_url`. | | `sourcegraph_url` | string | required by: `sourcegraph_enabled` | The Sourcegraph instance URL for integration. | @@ -357,5 +357,7 @@ are listed in the descriptions of the relevant settings. | `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider. | | `user_show_add_ssh_key_message` | boolean | no | When set to `false` disable the "You won't be able to pull or push project code via SSH" warning shown to users with no uploaded SSH key. | | `version_check_enabled` | boolean | no | Let GitLab inform you when an update is available. | -| `web_ide_clientside_preview_enabled` | boolean | no | Client side evaluation (allow live previews of JavaScript projects in the Web IDE using CodeSandbox client side evaluation). | +| `web_ide_clientside_preview_enabled` | boolean | no | Live Preview (allow live previews of JavaScript projects in the Web IDE using CodeSandbox Live Preview). | | `snippet_size_limit` | integer | no | Max snippet content size in **bytes**. Default: 52428800 Bytes (50MB).| +| `issues_create_limit` | integer | no | Max number of issue creation requests per minute per user. Default: 300. To disable throttling set to 0.| +| `raw_blob_request_limit` | integer | no | Max number of requests per minute for each raw path. Default: 300. To disable throttling set to 0.| diff --git a/doc/api/snippets.md b/doc/api/snippets.md index 0b41ab557ad..e2e39de412b 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -21,7 +21,7 @@ Valid values for snippet visibility levels are: Get a list of the current user's snippets. -```text +```plaintext GET /snippets ``` @@ -47,13 +47,13 @@ Example response: "username": "user0", "state": "active", "avatar_url": "https://www.gravatar.com/avatar/52e4ce24a915fb7e51e1ad3b57f4b00a?s=80&d=identicon", - "web_url": "http://localhost:3000/user0" + "web_url": "http://example.com/user0" }, "updated_at": "2018-09-18T01:12:26.383Z", "created_at": "2018-09-18T01:12:26.383Z", "project_id": null, - "web_url": "http://localhost:3000/snippets/42", - "raw_url": "http://localhost:3000/snippets/42/raw" + "web_url": "http://example.com/snippets/42", + "raw_url": "http://example.com/snippets/42/raw" }, { "id": 41, @@ -67,13 +67,13 @@ Example response: "username": "user0", "state": "active", "avatar_url": "https://www.gravatar.com/avatar/52e4ce24a915fb7e51e1ad3b57f4b00a?s=80&d=identicon", - "web_url": "http://localhost:3000/user0" + "web_url": "http://example.com/user0" }, "updated_at": "2018-09-18T01:12:26.360Z", "created_at": "2018-09-18T01:12:26.360Z", - "project_id": null, - "web_url": "http://localhost:3000/snippets/41", - "raw_url": "http://localhost:3000/snippets/41/raw" + "project_id": 1, + "web_url": "http://example.com/gitlab-org/gitlab-test/snippets/41", + "raw_url": "http://example.com/gitlab-org/gitlab-test/snippets/41/raw" } ] ``` @@ -82,7 +82,7 @@ Example response: Get a single snippet. -```text +```plaintext GET /snippets/:id ``` @@ -118,7 +118,9 @@ Example response: "expires_at": null, "updated_at": "2012-06-28T10:52:04Z", "created_at": "2012-06-28T10:52:04Z", + "project_id": null, "web_url": "http://example.com/snippets/1", + "raw_url": "http://example.com/snippets/1/raw" } ``` @@ -126,7 +128,7 @@ Example response: Get a single snippet's raw contents. -```text +```plaintext GET /snippets/:id/raw ``` @@ -144,7 +146,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap Example response: -```text +```plaintext Hello World snippet ``` @@ -155,7 +157,7 @@ Create a new snippet. NOTE: **Note:** The user must have permission to create new snippets. -```text +```plaintext POST /snippets ``` @@ -199,7 +201,9 @@ Example response: "expires_at": null, "updated_at": "2012-06-28T10:52:04Z", "created_at": "2012-06-28T10:52:04Z", + "project_id": null, "web_url": "http://example.com/snippets/1", + "raw_url": "http://example.com/snippets/1/raw" } ``` @@ -210,7 +214,7 @@ Update an existing snippet. NOTE: **Note:** The user must have permission to change an existing snippet. -```text +```plaintext PUT /snippets/:id ``` @@ -255,7 +259,9 @@ Example response: "expires_at": null, "updated_at": "2012-06-28T10:52:04Z", "created_at": "2012-06-28T10:52:04Z", + "project_id": null, "web_url": "http://example.com/snippets/1", + "raw_url": "http://example.com/snippets/1/raw" } ``` @@ -263,7 +269,7 @@ Example response: Delete an existing snippet. -```text +```plaintext DELETE /snippets/:id ``` @@ -290,7 +296,7 @@ The following are possible return codes: List all public snippets. -```text +```plaintext GET /snippets/public ``` @@ -318,15 +324,16 @@ Example response: "name": "Libby Rolfson", "state": "active", "username": "elton_wehner", - "web_url": "http://localhost:3000/elton_wehner" + "web_url": "http://example.com/elton_wehner" }, "created_at": "2016-11-25T16:53:34.504Z", "file_name": "oconnerrice.rb", "id": 49, - "raw_url": "http://localhost:3000/snippets/49/raw", "title": "Ratione cupiditate et laborum temporibus.", "updated_at": "2016-11-25T16:53:34.504Z", - "web_url": "http://localhost:3000/snippets/49" + "project_id": null, + "web_url": "http://example.com/snippets/49", + "raw_url": "http://example.com/snippets/49/raw" }, { "author": { @@ -335,15 +342,16 @@ Example response: "name": "Llewellyn Flatley", "state": "active", "username": "adaline", - "web_url": "http://localhost:3000/adaline" + "web_url": "http://example.com/adaline" }, "created_at": "2016-11-25T16:53:34.479Z", "file_name": "muellershields.rb", "id": 48, - "raw_url": "http://localhost:3000/snippets/48/raw", "title": "Minus similique nesciunt vel fugiat qui ullam sunt.", "updated_at": "2016-11-25T16:53:34.479Z", - "web_url": "http://localhost:3000/snippets/48", + "project_id": null, + "web_url": "http://example.com/snippets/48", + "raw_url": "http://example.com/snippets/49/raw", "visibility": "public" } ] @@ -356,7 +364,7 @@ Example response: NOTE: **Note:** Available only for administrators. -```text +```plaintext GET /snippets/:id/user_agent_detail ``` diff --git a/doc/api/templates/gitignores.md b/doc/api/templates/gitignores.md index 3acd666ad66..6e2e3e2d07f 100644 --- a/doc/api/templates/gitignores.md +++ b/doc/api/templates/gitignores.md @@ -2,7 +2,7 @@ type: reference --- -# .gitignore API +# `.gitignore` API In GitLab, there is an API endpoint available for `.gitignore`. For more information on `gitignore`, see the diff --git a/doc/api/todos.md b/doc/api/todos.md index 2711766c634..cd6f2468cc6 100644 --- a/doc/api/todos.md +++ b/doc/api/todos.md @@ -101,7 +101,8 @@ Example Response: "target_url": "https://gitlab.example.com/gitlab-org/gitlab-foss/-/merge_requests/7", "body": "Dolores in voluptatem tenetur praesentium omnis repellendus voluptatem quaerat.", "state": "pending", - "created_at": "2016-06-17T07:52:35.225Z" + "created_at": "2016-06-17T07:52:35.225Z", + "updated_at": "2016-06-17T07:52:35.225Z" }, { "id": 98, @@ -174,7 +175,8 @@ Example Response: "target_url": "https://gitlab.example.com/gitlab-org/gitlab-foss/-/merge_requests/7", "body": "Dolores in voluptatem tenetur praesentium omnis repellendus voluptatem quaerat.", "state": "pending", - "created_at": "2016-06-17T07:49:24.624Z" + "created_at": "2016-06-17T07:49:24.624Z", + "updated_at": "2016-06-17T07:49:24.624Z" } ] ``` @@ -272,7 +274,8 @@ Example Response: "target_url": "https://gitlab.example.com/gitlab-org/gitlab-foss/-/merge_requests/7", "body": "Dolores in voluptatem tenetur praesentium omnis repellendus voluptatem quaerat.", "state": "done", - "created_at": "2016-06-17T07:52:35.225Z" + "created_at": "2016-06-17T07:52:35.225Z", + "updated_at": "2016-06-17T07:52:35.225Z" } ``` diff --git a/doc/api/users.md b/doc/api/users.md index 90aafcef035..28d233ce6b3 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -611,7 +611,7 @@ GET /users/:id_or_username/status | Attribute | Type | Required | Description | | ---------------- | ------ | -------- | ----------- | -| `id_or_username` | string | yes | The id or username of the user to get a status of | +| `id_or_username` | string | yes | The ID or username of the user to get a status of | ```shell curl "https://gitlab.example.com/users/janedoe/status" @@ -723,7 +723,7 @@ GET /users/:id_or_username/keys | Attribute | Type | Required | Description | | ---------------- | ------ | -------- | ----------- | -| `id_or_username` | string | yes | The id or username of the user to get the SSH keys for. | +| `id_or_username` | string | yes | The ID or username of the user to get the SSH keys for. | ## Single SSH key @@ -794,7 +794,7 @@ POST /users/:id/keys Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user - `title` (required) - new SSH Key's title - `key` (required) - new SSH key @@ -821,7 +821,7 @@ DELETE /users/:id/keys/:key_id Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user - `key_id` (required) - SSH key ID ## List all GPG keys @@ -1070,7 +1070,7 @@ GET /users/:id/emails Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user ## Single email @@ -1133,7 +1133,7 @@ POST /users/:id/emails Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user - `email` (required) - email address - `skip_confirmation` (optional) - Skip confirmation and assume e-mail is verified - true or false (default) @@ -1160,7 +1160,7 @@ DELETE /users/:id/emails/:email_id Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user - `email_id` (required) - email ID ## Block user @@ -1173,7 +1173,7 @@ POST /users/:id/block Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user Returns: @@ -1191,7 +1191,7 @@ POST /users/:id/unblock Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user Will return `201 OK` on success, `404 User Not Found` is user cannot be found or `403 Forbidden` when trying to unblock a user blocked by LDAP synchronization. @@ -1208,7 +1208,7 @@ POST /users/:id/deactivate Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user Returns: @@ -1230,7 +1230,7 @@ POST /users/:id/activate Parameters: -- `id` (required) - id of specified user +- `id` (required) - ID of specified user Returns: @@ -1409,6 +1409,7 @@ The activities that update the timestamp are: - User logging in into GitLab - User visiting pages related to Dashboards, Projects, Issues, and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/54947) in GitLab 11.8) - User using the API +- User using the GraphQL API By default, it shows the activity for all users in the last 6 months, but this can be amended by using the `from` parameter. diff --git a/doc/api/vulnerability_exports.md b/doc/api/vulnerability_exports.md index f2666783087..2c9ac5d65eb 100644 --- a/doc/api/vulnerability_exports.md +++ b/doc/api/vulnerability_exports.md @@ -1,6 +1,6 @@ -# Project Vulnerabilities API **(ULTIMATE)** +# Vulnerability export API **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197494) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197494) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. [Updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30397) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. CAUTION: **Caution:** This API is currently in development and is protected by a **disabled** @@ -17,21 +17,21 @@ across GitLab releases. Every API call to vulnerability exports must be [authenticated](README.md#authentication). +## Create a project-level vulnerability export + +Creates a new vulnerability export for a project. + Vulnerability export permissions inherit permissions from their project. If a project is private and a user isn't a member of the project to which the vulnerability belongs, requests to that project return a `404 Not Found` status code. Vulnerability exports can be only accessed by the export's author. -## Create vulnerability export - -Creates a new vulnerability export. - If an authenticated user doesn't have permission to [create a new vulnerability](../user/permissions.md#project-members-permissions), this request results in a `403` status code. ```plaintext -POST /projects/:id/vulnerability_exports +POST /security/projects/:id/vulnerability_exports ``` | Attribute | Type | Required | Description | @@ -39,10 +39,10 @@ POST /projects/:id/vulnerability_exports | `id` | integer or string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the project which the authenticated user is a member of | ```shell -curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/vulnerability_exports +curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/projects/1/vulnerability_exports ``` -The created vulnerability export will be automatically deleted after 1 hour. +The created vulnerability export is automatically deleted after 1 hour. Example response: @@ -51,13 +51,93 @@ Example response: "id": 2, "created_at": "2020-03-30T09:35:38.746Z", "project_id": 1, + "group_id": null, "format": "csv", "status": "created", "started_at": null, "finished_at": null, "_links": { - "self": "https://gitlab.example.com/api/v4/projects/1/vulnerability_exports/2", - "download": "https://gitlab.example.com/api/v4/projects/1/vulnerability_exports/2/download" + "self": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2", + "download": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2/download" + } +} +``` + +## Create a group-level vulnerability export + +Creates a new vulnerability export for a group. + +Vulnerability export permissions inherit permissions from their group. If a group is +private and a user isn't a member of the group to which the vulnerability +belongs, requests to that group return a `404 Not Found` status code. +Vulnerability exports can be only accessed by the export's author. + +If an authenticated user doesn't have permission to +[create a new vulnerability](../user/permissions.md#group-members-permissions), +this request results in a `403` status code. + +```plaintext +POST /security/groups/:id/vulnerability_exports +``` + +| Attribute | Type | Required | Description | +| ------------------- | ----------------- | ---------- | -----------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the group which the authenticated user is a member of | + +```shell +curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/groups/1/vulnerability_exports +``` + +The created vulnerability export is automatically deleted after 1 hour. + +Example response: + +```json +{ + "id": 2, + "created_at": "2020-03-30T09:35:38.746Z", + "project_id": null, + "group_id": 1, + "format": "csv", + "status": "created", + "started_at": null, + "finished_at": null, + "_links": { + "self": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2", + "download": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2/download" + } +} +``` + +## Create an instance-level vulnerability export + +Creates a new vulnerability export for the projects of the user selected in the Security Dashboard. + +```plaintext +POST /security/vulnerability_exports +``` + +```shell +curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/vulnerability_exports +``` + +The created vulnerability export is automatically deleted after one hour. + +Example response: + +```json +{ + "id": 2, + "created_at": "2020-03-30T09:35:38.746Z", + "project_id": null, + "group_id": null, + "format": "csv", + "status": "created", + "started_at": null, + "finished_at": null, + "_links": { + "self": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2", + "download": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2/download" } } ``` @@ -67,16 +147,15 @@ Example response: Gets a single vulnerability export. ```plaintext -POST /projects/:id/vulnerability_exports/:vulnerability_export_id +GET /security/vulnerability_exports/:id ``` | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer or string | yes | The vulnerability's ID | -| `vulnerability_export_id` | integer or string | yes | The vulnerability export's ID | +| `id` | integer or string | yes | The vulnerability export's ID | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/vulnerability_exports/2 +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/vulnerability_exports/2 ``` If the vulnerability export isn't finished, the response is `202 Accepted`. @@ -88,13 +167,14 @@ Example response: "id": 2, "created_at": "2020-03-30T09:35:38.746Z", "project_id": 1, + "group_id": null, "format": "csv", "status": "finished", "started_at": "2020-03-30T09:36:54.469Z", "finished_at": "2020-03-30T09:36:55.008Z", "_links": { - "self": "https://gitlab.example.com/api/v4/projects/1/vulnerability_exports/2", - "download": "https://gitlab.example.com/api/v4/projects/1/vulnerability_exports/2/download" + "self": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2", + "download": "https://gitlab.example.com/api/v4/security/vulnerability_exports/2/download" } } ``` @@ -104,16 +184,15 @@ Example response: Downloads a single vulnerability export. ```plaintext -POST /projects/:id/vulnerability_exports/:vulnerability_export_id/download +GET /security/vulnerability_exports/:id/download ``` | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer or string | yes | The vulnerability's ID | -| `vulnerability_export_id` | integer or string | yes | The vulnerability export's ID | +| `id` | integer or string | yes | The vulnerability export's ID | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/vulnerability_exports/2/download +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/vulnerability_exports/2/download ``` The response will be `404 Not Found` if the vulnerability export is not finished yet or was not found. @@ -121,18 +200,18 @@ The response will be `404 Not Found` if the vulnerability export is not finished Example response: ```csv -Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE -container_scanning,Clair,confirmed,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997 -container_scanning,Clair,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269 -container_scanning,Clair,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001 -container_scanning,Clair,detected,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228 -container_scanning,Clair,confirmed,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052 -container_scanning,Clair,detected,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520 -container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869 -dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a -dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98 -sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47 -sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29 -sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41 -sast,Find Security Bugs,confirmed,ECB mode is insecure 2,,ECB mode is insecure,medium,ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29 -``` +Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE +Gitlab.org,Defend,container_scanning,Clair,confirmed,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997 +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269 +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001 +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228 +Gitlab.org,Defend,container_scanning,Clair,confirmed,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052 +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520 +Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869 +Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a +Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98 +Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47 +Gitlab.org,Defend,sast,Find Security Bugs,detected,Cipher with no integrity,,Cipher with no integrity,medium,e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29 +Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41 +Gitlab.org,Defend,sast,Find Security Bugs,confirmed,ECB mode is insecure 2,,ECB mode is insecure,medium,ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29 +Gitlab.org,Defend,``` diff --git a/doc/api/wikis.md b/doc/api/wikis.md index cdaf95fc291..48b04fefd39 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -1,6 +1,6 @@ # Wikis API -> [Introduced][ce-13372] in GitLab 10.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13372) in GitLab 10.0. Available only in APIv4. @@ -153,8 +153,6 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git On success the HTTP status code is `204` and no JSON response is expected. -[ce-13372]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13372 - ## Upload an attachment to the wiki repository Uploads a file to the attachment folder inside the wiki's repository. The diff --git a/doc/ci/README.md b/doc/ci/README.md index 30971d422cc..fce0ad15b70 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -37,14 +37,17 @@ your app. For a complete overview of these methodologies and GitLab CI/CD, read the [Introduction to CI/CD with GitLab](introduction/index.md). -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For a video demonstration of GitLab CI/CD, see [Demo: CI/CD with GitLab](https://www.youtube.com/watch?v=1iXFbchozdY). +<div class="video-fallback"> + Video demonstration of GitLab CI/CD: <a href="https://www.youtube.com/watch?v=1iXFbchozdY">Demo: CI/CD with GitLab</a>. +</div> +<figure class="video-container"> + <iframe src="https://www.youtube.com/embed/1iXFbchozdY" frameborder="0" allowfullscreen="true"> </iframe> +</figure> ## Getting started GitLab CI/CD is configured by a file called `.gitlab-ci.yml` placed -at the repository's root. The scripts set in this file are executed -by the [GitLab Runner](https://docs.gitlab.com/runner/). +at the repository's root. This file creates a [pipeline](pipelines/index.md), which runs for changes to the code in the repository. Pipelines consist of one or more stages that run in order and can each contain one or more jobs that run in parallel. These jobs (or scripts) get executed by the [GitLab Runner](https://docs.gitlab.com/runner/) agent. To get started with GitLab CI/CD, we recommend you read through the following documents: @@ -74,25 +77,32 @@ for all the attributes you can set and use. NOTE: **Note:** GitLab CI/CD and [shared runners](runners/README.md#shared-specific-and-group-runners) are enabled in GitLab.com and available for all users, limited only to the [user's pipelines quota](../user/gitlab_com/index.md#shared-runners). -## Configuration +## Concepts -GitLab CI/CD supports numerous configuration options: +GitLab CI/CD uses a number of concepts to describe and run your build and deploy. -| Configuration | Description | +| Concept | Description | |:--------------|:-------------| | [Pipelines](pipelines/index.md) | Structure your CI/CD process through pipelines. | | [Environment variables](variables/README.md) | Reuse values based on a variable/value key pair. | -| [Environments](environments.md) | Deploy your application to different environments (e.g., staging, production). | +| [Environments](environments/index.md) | Deploy your application to different environments (e.g., staging, production). | | [Job artifacts](pipelines/job_artifacts.md) | Output, use, and reuse job artifacts. | | [Cache dependencies](caching/index.md) | Cache your dependencies for a faster execution. | +| [GitLab Runner](https://docs.gitlab.com/runner/) | Configure your own GitLab Runners to execute your scripts. | + +## Configuration + +GitLab CI/CD supports numerous configuration options: + +| Configuration | Description | +|:--------------|:-------------| | [Schedule pipelines](pipelines/schedules.md) | Schedule pipelines to run as often as you need. | | [Custom path for `.gitlab-ci.yml`](pipelines/settings.md#custom-ci-configuration-path) | Define a custom path for the CI/CD configuration file. | | [Git submodules for CI/CD](git_submodules.md) | Configure jobs for using Git submodules.| | [SSH keys for CI/CD](ssh_keys/README.md) | Using SSH keys in your CI pipelines. | -| [Pipelines triggers](triggers/README.md) | Trigger pipelines through the API. | +| [Pipeline triggers](triggers/README.md) | Trigger pipelines through the API. | | [Pipelines for Merge Requests](merge_request_pipelines/index.md) | Design a pipeline structure for running a pipeline in merge requests. | | [Integrate with Kubernetes clusters](../user/project/clusters/index.md) | Connect your project to Google Kubernetes Engine (GKE) or an existing Kubernetes cluster. | -| [GitLab Runner](https://docs.gitlab.com/runner/) | Configure your own GitLab Runners to execute your scripts. | | [Optimize GitLab and Runner for large repositories](large_repositories/index.md) | Recommended strategies for handling large repos. | | [`.gitlab-ci.yml` full reference](yaml/README.md) | All the attributes you can use with GitLab CI/CD. | diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index 12267b4ab9f..16cabae353e 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -39,8 +39,9 @@ runtime dependencies needed to compile the project: - `artifacts`: **Use for stage results that will be passed between stages.** Artifacts are files generated by a job which are stored and uploaded, and can then - be fetched and used by jobs in later stages of the **same pipeline**. This data - will not be available in different pipelines, but is available to be downloaded + be fetched and used by jobs in later stages of the **same pipeline**. In other words, + [you can't create an artifact in job-A in stage-1, and then use this artifact in job-B in stage-1](https://gitlab.com/gitlab-org/gitlab/-/issues/25837). + This data will not be available in different pipelines, but is available to be downloaded from the UI. The name `artifacts` sounds like it's only useful outside of the job, like for downloading diff --git a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md index 3437f867cb8..2836f9932c6 100644 --- a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md +++ b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md @@ -29,7 +29,7 @@ To use GitLab CI/CD with a Bitbucket Cloud repository: The web hook URL should be set to the GitLab API to trigger pull mirroring, using the Personal Access Token we just generated for authentication. - ```text + ```plaintext https://gitlab.com/api/v4/projects/<PROJECT_ID>/mirror/pull?private_token=<PERSONAL_ACCESS_TOKEN> ``` diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index f70998a5f49..8883ad5dfd6 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -36,7 +39,7 @@ Some credentials are required to be able to run `aws` commands: 1. You can now use `aws` commands in the `.gitlab-ci.yml` file of this project: - ```yml + ```yaml deploy: stage: deploy image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest # see the note below diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index d4b87648f49..8722efd3b40 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -4,7 +4,8 @@ type: reference # Directed Acyclic Graph -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47063) in GitLab 12.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47063) in GitLab 12.2. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/206902) in GitLab 12.10. A [directed acyclic graph](https://www.techopedia.com/definition/5739/directed-acyclic-graph-dag) can be used in the context of a CI/CD pipeline to build relationships between jobs such that diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 427f61deb29..f992af6c8a5 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -476,7 +476,7 @@ which can be avoided if a different driver is used, for example `overlay2`. On Ubuntu systems, this is done by editing `/etc/modules`. Just add the following line into it: - ```text + ```plaintext overlay ``` diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 90e5c77063e..51139da2d16 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -26,7 +26,20 @@ test them on a dedicated CI server. To use GitLab Runner with Docker you need to [register a new Runner](https://docs.gitlab.com/runner/register/) to use the `docker` executor. -A one-line example can be seen below: +An example can be seen below. First we set up a temporary template to supply the services: + +```shell +cat > /tmp/test-config.template.toml << EOF +[[runners]] +[runners.docker] +[[runners.docker.services]] +name = "postgres:latest" +[[runners.docker.services]] +name = "mysql:latest" +EOF +``` + +Then we register the runner using the template that was just created: ```shell sudo gitlab-runner register \ @@ -34,9 +47,8 @@ sudo gitlab-runner register \ --registration-token "PROJECT_REGISTRATION_TOKEN" \ --description "docker-ruby:2.6" \ --executor "docker" \ - --docker-image ruby:2.6 \ - --docker-services postgres:latest \ - --docker-services mysql:latest + --template-config /tmp/test-config.template.toml \ + --docker-image ruby:2.6 ``` The registered runner will use the `ruby:2.6` Docker image and will run two @@ -197,7 +209,7 @@ default: image: ruby:2.6 services: - - postgres:9.3 + - postgres:11.7 before_script: - bundle install @@ -207,6 +219,12 @@ test: - bundle exec rake spec ``` +The image name must be in one of the following formats: + +- `image: <image-name>` (Same as using `<image-name>` with the `latest` tag) +- `image: <image-name>:<tag>` +- `image: <image-name>@<digest>` + It is also possible to define different images and services per job: ```yaml @@ -217,14 +235,14 @@ default: test:2.6: image: ruby:2.6 services: - - postgres:9.3 + - postgres:11.7 script: - bundle exec rake spec test:2.7: image: ruby:2.7 services: - - postgres:9.4 + - postgres:12.2 script: - bundle exec rake spec ``` @@ -239,7 +257,7 @@ default: entrypoint: ["/bin/bash"] services: - - name: my-postgres:9.4 + - name: my-postgres:11.7 alias: db-postgres entrypoint: ["/usr/local/bin/db-postgres"] command: ["start"] @@ -271,7 +289,7 @@ variables: POSTGRES_INITDB_ARGS: "--encoding=UTF8 --data-checksums" services: -- name: postgres:9.4 +- name: postgres:11.7 alias: db entrypoint: ["docker-entrypoint.sh"] command: ["postgres"] diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index 5a6f7490266..587f1f91f72 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -82,6 +82,7 @@ store: ```yaml before_script: + - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - | echo "-----BEGIN CERTIFICATE----- @@ -89,6 +90,18 @@ store: -----END CERTIFICATE-----" >> /kaniko/ssl/certs/ca-certificates.crt ``` +## Video walkthrough of a working example + +The [Least Privilege Container Builds with Kaniko on GitLab](https://www.youtube.com/watch?v=d96ybcELpFs) +video is a walkthrough of the [Kaniko Docker Build](https://gitlab.com/guided-explorations/containers/kaniko-docker-build) +Guided Exploration project pipeline. It was tested on: + +- [GitLab.com Shared Runners](../../user/gitlab_com/index.md#shared-runners) +- [The Kubernetes Runner executor](https://docs.gitlab.com/runner/executors/kubernetes.html) + +The example can be copied to your own group or instance for testing. More details +on what other GitLab CI patterns are demonstrated are available at the project page. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/ci/environments.md b/doc/ci/environments.md index fdd2791aa1d..b1dc9af6b5b 100644 --- a/doc/ci/environments.md +++ b/doc/ci/environments.md @@ -1,984 +1,5 @@ --- -type: reference +redirect_to: 'environments/index.md' --- -# Environments and deployments - -> Introduced in GitLab 8.9. - -Environments allow control of the continuous deployment of your software, -all within GitLab. - -## Introduction - -There are many stages required in the software development process before the software is ready -for public consumption. - -For example: - -1. Develop your code. -1. Test your code. -1. Deploy your code into a testing or staging environment before you release it to the public. - -This helps find bugs in your software, and also in the deployment process as well. - -GitLab CI/CD is capable of not only testing or building your projects, but also -deploying them in your infrastructure, with the added benefit of giving you a -way to track your deployments. In other words, you will always know what is -currently being deployed or has been deployed on your servers. - -It's important to know that: - -- Environments are like tags for your CI jobs, describing where code gets deployed. -- Deployments are created when [jobs](yaml/README.md#introduction) deploy versions of code to environments, - so every environment can have one or more deployments. - -GitLab: - -- Provides a full history of your deployments for each environment. -- Keeps track of your deployments, so you always know what is currently being deployed on your - servers. - -If you have a deployment service such as [Kubernetes](../user/project/clusters/index.md) -associated with your project, you can use it to assist with your deployments, and -can even access a [web terminal](#web-terminals) for your environment from within GitLab! - -## Configuring environments - -Configuring environments involves: - -1. Understanding how [pipelines](pipelines/index.md) work. -1. Defining environments in your project's [`.gitlab-ci.yml`](yaml/README.md) file. -1. Creating a job configured to deploy your application. For example, a deploy job configured with [`environment`](yaml/README.md#environment) to deploy your application to a [Kubernetes cluster](../user/project/clusters/index.md). - -The rest of this section illustrates how to configure environments and deployments using -an example scenario. It assumes you have already: - -- Created a [project](../gitlab-basics/create-project.md) in GitLab. -- Set up [a Runner](runners/README.md). - -In the scenario: - -- We are developing an application. -- We want to run tests and build our app on all branches. -- Our default branch is `master`. -- We deploy the app only when a pipeline on `master` branch is run. - -### Defining environments - -Let's consider the following `.gitlab-ci.yml` example: - -```yaml -stages: - - test - - build - - deploy - -test: - stage: test - script: echo "Running tests" - -build: - stage: build - script: echo "Building the app" - -deploy_staging: - stage: deploy - script: - - echo "Deploy to staging server" - environment: - name: staging - url: https://staging.example.com - only: - - master -``` - -We have defined three [stages](yaml/README.md#stages): - -- `test` -- `build` -- `deploy` - -The jobs assigned to these stages will run in this order. If any job fails, then -the pipeline fails and jobs that are assigned to the next stage won't run. - -In our case: - -- The `test` job will run first. -- Then the `build` job. -- Lastly the `deploy_staging` job. - -With this configuration, we: - -- Check that the tests pass. -- Ensure that our app is able to be built successfully. -- Lastly we deploy to the staging server. - -NOTE: **Note:** -The `environment` keyword defines where the app is deployed. -The environment `name` and `url` is exposed in various places -within GitLab. Each time a job that has an environment specified -succeeds, a deployment is recorded, along with the Git SHA, and environment name. - -CAUTION: **Caution**: -Some characters are not allowed in environment names. Use only letters, -numbers, spaces, and `-`, `_`, `/`, `{`, `}`, or `.`. Also, it must not start nor end with `/`. - -In summary, with the above `.gitlab-ci.yml` we have achieved the following: - -- All branches will run the `test` and `build` jobs. -- The `deploy_staging` job will run [only](yaml/README.md#onlyexcept-basic) on the `master` - branch, which means all merge requests that are created from branches don't - get deployed to the staging server. -- When a merge request is merged, all jobs will run and the `deploy_staging` - job will deploy our code to a staging server while the deployment - will be recorded in an environment named `staging`. - -#### Environment variables and Runner - -Starting with GitLab 8.15, the environment name is exposed to the Runner in -two forms: - -- `$CI_ENVIRONMENT_NAME`. The name given in `.gitlab-ci.yml` (with any variables - expanded). -- `$CI_ENVIRONMENT_SLUG`. A "cleaned-up" version of the name, suitable for use in URLs, - DNS, etc. - -If you change the name of an existing environment, the: - -- `$CI_ENVIRONMENT_NAME` variable will be updated with the new environment name. -- `$CI_ENVIRONMENT_SLUG` variable will remain unchanged to prevent unintended side - effects. - -Starting with GitLab 9.3, the environment URL is exposed to the Runner via -`$CI_ENVIRONMENT_URL`. The URL is expanded from either: - -- `.gitlab-ci.yml`. -- The external URL from the environment if not defined in `.gitlab-ci.yml`. - -#### Set dynamic environment URLs after a job finishes - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. - -In a job script, you can specify a static [environment URL](#using-the-environment-url). -However, there may be times when you want a dynamic URL. For example, -if you deploy a Review App to an external hosting -service that generates a random URL per deployment, like `https://94dd65b.amazonaws.com/qa-lambda-1234567`, -you don't know the URL before the deployment script finishes. -If you want to use the environment URL in GitLab, you would have to update it manually. - -To address this problem, you can configure a deployment job to report back a set of -variables, including the URL that was dynamically-generated by the external service. -GitLab supports [dotenv](https://github.com/bkeepers/dotenv) file as the format, -and expands the `environment:url` value with variables defined in the dotenv file. - -To use this feature, specify the -[`artifacts:reports:dotenv`](yaml/README.md#artifactsreportsdotenv) keyword in `.gitlab-ci.yml`. - -##### Example of setting dynamic environment URLs - -The following example shows a Review App that creates a new environment -per merge request. The `review` job is triggered by every push, and -creates or updates an environment named `review/your-branch-name`. -The environment URL is set to `$DYNAMIC_ENVIRONMENT_URL`: - -```yaml -review: - script: - - DYNAMIC_ENVIRONMENT_URL=$(deploy-script) # In script, get the environment URL. - - echo "DYNAMIC_ENVIRONMENT_URL=$DYNAMIC_ENVIRONMENT_URL" >> deploy.env # Add the value to a dotenv file. - artifacts: - reports: - dotenv: deploy.env # Report back dotenv file to rails. - environment: - name: review/$CI_COMMIT_REF_SLUG - url: $DYNAMIC_ENVIRONMENT_URL # and set the variable produced in script to `environment:url` - on_stop: stop_review - -stop_review: - script: - - ./teardown-environment - when: manual - environment: - name: review/$CI_COMMIT_REF_SLUG - action: stop -``` - -As soon as the `review` job finishes, GitLab updates the `review/your-branch-name` -environment's URL. -It parses the report artifact `deploy.env`, registers a list of variables as runtime-created, -uses it for expanding `environment:url: $DYNAMIC_ENVIRONMENT_URL` and sets it to the environment URL. -You can also specify a static part of the URL at `environment:url:`, such as -`https://$DYNAMIC_ENVIRONMENT_URL`. If the value of `DYNAMIC_ENVIRONMENT_URL` is -`123.awesome.com`, the final result will be `https://123.awesome.com`. - -The assigned URL for the `review/your-branch-name` environment is visible in the UI. -[See where the environment URL is displayed](#using-the-environment-url). - -> **Notes:** -> -> - `stop_review` doesn't generate a dotenv report artifact, so it won't recognize the `DYNAMIC_ENVIRONMENT_URL` variable. Therefore you should not set `environment:url:` in the `stop_review` job. -> - If the environment URL is not valid (for example, the URL is malformed), the system doesn't update the environment URL. - -### Configuring manual deployments - -Adding `when: manual` to an automatically executed job's configuration converts it to -a job requiring manual action. - -To expand on the [previous example](#defining-environments), the following includes -another job that deploys our app to a production server and is -tracked by a `production` environment. - -The `.gitlab-ci.yml` file for this is as follows: - -```yaml -stages: - - test - - build - - deploy - -test: - stage: test - script: echo "Running tests" - -build: - stage: build - script: echo "Building the app" - -deploy_staging: - stage: deploy - script: - - echo "Deploy to staging server" - environment: - name: staging - url: https://staging.example.com - only: - - master - -deploy_prod: - stage: deploy - script: - - echo "Deploy to production server" - environment: - name: production - url: https://example.com - when: manual - only: - - master -``` - -The `when: manual` action: - -- Exposes a "play" button in GitLab's UI for that job. -- Means the `deploy_prod` job will only be triggered when the "play" button is clicked. - -You can find the "play" button in the pipelines, environments, deployments, and jobs views. - -| View | Screenshot | -|:----------------|:-------------------------------------------------------------------------------| -| Pipelines |  | -| Single pipeline |  | -| Environments |  | -| Deployments |  | -| Jobs |  | - -Clicking on the play button in any view will trigger the `deploy_prod` job, and the -deployment will be recorded as a new environment named `production`. - -NOTE: **Note:** -If your environment's name is `production` (all lowercase), -it will get recorded in [Value Stream Analytics](../user/project/cycle_analytics.md). - -### Configuring dynamic environments - -Regular environments are good when deploying to "stable" environments like staging or production. - -However, for environments for branches other than `master`, dynamic environments -can be used. Dynamic environments make it possible to create environments on the fly by -declaring their names dynamically in `.gitlab-ci.yml`. - -Dynamic environments are a fundamental part of [Review apps](review_apps/index.md). - -### Configuring incremental rollouts - -Learn how to release production changes to only a portion of your Kubernetes pods with -[incremental rollouts](environments/incremental_rollouts.md). - -#### Allowed variables - -The `name` and `url` parameters for dynamic environments can use most available CI/CD variables, -including: - -- [Predefined environment variables](variables/README.md#predefined-environment-variables) -- [Project and group variables](variables/README.md#gitlab-cicd-environment-variables) -- [`.gitlab-ci.yml` variables](yaml/README.md#variables) - -However, you cannot use variables defined: - -- Under `script`. -- On the Runner's side. - -There are also other variables that are unsupported in the context of `environment:name`. -For more information, see [Where variables can be used](variables/where_variables_can_be_used.md). - -#### Example configuration - -GitLab Runner exposes various [environment variables](variables/README.md) when a job runs, so -you can use them as environment names. - -In the following example, the job will deploy to all branches except `master`: - -```yaml -deploy_review: - stage: deploy - script: - - echo "Deploy a review app" - environment: - name: review/$CI_COMMIT_REF_NAME - url: https://$CI_ENVIRONMENT_SLUG.example.com - only: - - branches - except: - - master -``` - -In this example: - -- The job's name is `deploy_review` and it runs on the `deploy` stage. -- We set the `environment` with the `environment:name` as `review/$CI_COMMIT_REF_NAME`. - Since the [environment name](yaml/README.md#environmentname) can contain slashes (`/`), we can - use this pattern to distinguish between dynamic and regular environments. -- We tell the job to run [`only`](yaml/README.md#onlyexcept-basic) on branches, - [`except`](yaml/README.md#onlyexcept-basic) `master`. - -For the value of: - -- `environment:name`, the first part is `review`, followed by a `/` and then `$CI_COMMIT_REF_NAME`, - which receives the value of the branch name. -- `environment:url`, we want a specific and distinct URL for each branch. `$CI_COMMIT_REF_NAME` - may contain a `/` or other characters that would be invalid in a domain name or URL, - so we use `$CI_ENVIRONMENT_SLUG` to guarantee that we get a valid URL. - - For example, given a `$CI_COMMIT_REF_NAME` of `100-Do-The-Thing`, the URL will be something - like `https://100-do-the-4f99a2.example.com`. Again, the way you set up - the web server to serve these requests is based on your setup. - - We have used `$CI_ENVIRONMENT_SLUG` here because it is guaranteed to be unique. If - you're using a workflow like [GitLab Flow](../topics/gitlab_flow.md), collisions - are unlikely and you may prefer environment names to be more closely based on the - branch name. In that case, you could use `$CI_COMMIT_REF_NAME` in `environment:url` in - the example above: `https://$CI_COMMIT_REF_NAME.example.com`, which would give a URL - of `https://100-do-the-thing.example.com`. - -NOTE: **Note:** -You are not required to use the same prefix or only slashes (`/`) in the dynamic environments' -names. However, using this format will enable the [grouping similar environments](#grouping-similar-environments) -feature. - -### Configuring Kubernetes deployments - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6. - -If you are deploying to a [Kubernetes cluster](../user/project/clusters/index.md) -associated with your project, you can configure these deployments from your -`gitlab-ci.yml` file. - -The following configuration options are supported: - -- [`namespace`](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) - -In the following example, the job will deploy your application to the -`production` Kubernetes namespace. - -```yaml -deploy: - stage: deploy - script: - - echo "Deploy to production server" - environment: - name: production - url: https://example.com - kubernetes: - namespace: production - only: - - master -``` - -When deploying to a Kubernetes cluster using GitLab's Kubernetes integration, -information about the cluster and namespace will be displayed above the job -trace on the deployment job page: - - - -NOTE: **Note:** -Kubernetes configuration is not supported for Kubernetes clusters -that are [managed by GitLab](../user/project/clusters/index.md#gitlab-managed-clusters). -To follow progress on support for GitLab-managed clusters, see the -[relevant issue](https://gitlab.com/gitlab-org/gitlab/issues/38054). - -### Complete example - -The configuration in this section provides a full development workflow where your app is: - -- Tested. -- Built. -- Deployed as a Review App. -- Deployed to a staging server once the merge request is merged. -- Finally, able to be manually deployed to the production server. - -The following combines the previous configuration examples, including: - -- Defining [simple environments](#defining-environments) for testing, building, and deployment to staging. -- Adding [manual actions](#configuring-manual-deployments) for deployment to production. -- Creating [dynamic environments](#configuring-dynamic-environments) for deployments for reviewing. - -```yaml -stages: - - test - - build - - deploy - -test: - stage: test - script: echo "Running tests" - -build: - stage: build - script: echo "Building the app" - -deploy_review: - stage: deploy - script: - - echo "Deploy a review app" - environment: - name: review/$CI_COMMIT_REF_NAME - url: https://$CI_ENVIRONMENT_SLUG.example.com - only: - - branches - except: - - master - -deploy_staging: - stage: deploy - script: - - echo "Deploy to staging server" - environment: - name: staging - url: https://staging.example.com - only: - - master - -deploy_prod: - stage: deploy - script: - - echo "Deploy to production server" - environment: - name: production - url: https://example.com - when: manual - only: - - master -``` - -A more realistic example would also include copying files to a location where a -webserver (for example, NGINX) could then access and serve them. - -The example below will copy the `public` directory to `/srv/nginx/$CI_COMMIT_REF_SLUG/public`: - -```yaml -review_app: - stage: deploy - script: - - rsync -av --delete public /srv/nginx/$CI_COMMIT_REF_SLUG - environment: - name: review/$CI_COMMIT_REF_NAME - url: https://$CI_COMMIT_REF_SLUG.example.com -``` - -This example requires that NGINX and GitLab Runner are set up on the server this job will run on. - -NOTE: **Note:** -See the [limitations](#limitations) section for some edge cases regarding the naming of -your branches and Review Apps. - -The complete example provides the following workflow to developers: - -- Create a branch locally. -- Make changes and commit them. -- Push the branch to GitLab. -- Create a merge request. - -Behind the scenes, GitLab Runner will: - -- Pick up the changes and start running the jobs. -- Run the jobs sequentially as defined in `stages`: - - First, run the tests. - - If the tests succeed, build the app. - - If the build succeeds, the app is deployed to an environment with a name specific to the - branch. - -So now, every branch: - -- Gets its own environment. -- Is deployed to its own unique location, with the added benefit of: - - Having a [history of deployments](#viewing-deployment-history). - - Being able to [rollback changes](#retrying-and-rolling-back) if needed. - -For more information, see [Using the environment URL](#using-the-environment-url). - -### Protected environments - -Environments can be "protected", restricting access to them. - -For more information, see [Protected environments](environments/protected_environments.md). - -## Working with environments - -Once environments are configured, GitLab provides many features for working with them, -as documented below. - -### Viewing environments and deployments - -A list of environments and deployment statuses is available on each project's **Operations > Environments** page. - -For example: - - - -This example shows: - -- The environment's name with a link to its deployments. -- The last deployment ID number and who performed it. -- The job ID of the last deployment with its respective job name. -- The commit information of the last deployment, such as who committed it, to what - branch, and the Git SHA of the commit. -- The exact time the last deployment was performed. -- A button that takes you to the URL that you defined under the `environment` keyword - in `.gitlab-ci.yml`. -- A button that re-deploys the latest deployment, meaning it runs the job - defined by the environment name for that specific commit. - -The information shown in the **Environments** page is limited to the latest -deployments, but an environment can have multiple deployments. - -> **Notes:** -> -> - While you can create environments manually in the web interface, we recommend -> that you define your environments in `.gitlab-ci.yml` first. They will -> be automatically created for you after the first deploy. -> - The environments page can only be viewed by users with [Reporter permission](../user/permissions.md#project-members-permissions) -> and above. For more information on permissions, see the [permissions documentation](../user/permissions.md). -> - Only deploys that happen after your `.gitlab-ci.yml` is properly configured -> will show up in the **Environment** and **Last deployment** lists. - -### Viewing deployment history - -GitLab keeps track of your deployments, so you: - -- Always know what is currently being deployed on your servers. -- Can have the full history of your deployments for every environment. - -Clicking on an environment shows the history of its deployments. Here's an example **Environments** page -with multiple deployments: - - - -This view is similar to the **Environments** page, but all deployments are shown. Also in this view -is a **Rollback** button. For more information, see [Retrying and rolling back](#retrying-and-rolling-back). - -### Retrying and rolling back - -If there is a problem with a deployment, you can retry it or roll it back. - -To retry or rollback a deployment: - -1. Navigate to **Operations > Environments**. -1. Click on the environment. -1. In the deployment history list for the environment, click the: - - **Retry** button next to the last deployment, to retry that deployment. - - **Rollback** button next to a previously successful deployment, to roll back to that deployment. - -#### What to expect with a rollback - -Pressing the **Rollback** button on a specific commit will trigger a _new_ deployment with its -own unique job ID. - -This means that you will see a new deployment that points to the commit you are rolling back to. - -NOTE: **Note:** -The defined deployment process in the job's `script` determines whether the rollback succeeds or not. - -### Using the environment URL - -The [environment URL](yaml/README.md#environmenturl) is exposed in a few -places within GitLab: - -- In a merge request widget as a link: -  -- In the Environments view as a button: -  -- In the Deployments view as a button: -  - -You can see this information in a merge request itself if: - -- The merge request is eventually merged to the default branch (usually `master`). -- That branch also deploys to an environment (for example, `staging` or `production`). - -For example: - - - -#### Going from source files to public pages - -With GitLab's [Route Maps](review_apps/index.md#route-maps) you can go directly -from source files to public pages in the environment set for Review Apps. - -### Stopping an environment - -Stopping an environment: - -- Moves it from the list of **Available** environments to the list of **Stopped** - environments on the [**Environments** page](#viewing-environments-and-deployments). -- Executes an [`on_stop` action](yaml/README.md#environmenton_stop), if defined. - -This is often used when multiple developers are working on a project at the same time, -each of them pushing to their own branches, causing many dynamic environments to be created. - -NOTE: **Note:** -Starting with GitLab 8.14, dynamic environments are stopped automatically -when their associated branch is deleted. - -#### Automatically stopping an environment - -Environments can be stopped automatically using special configuration. - -Consider the following example where the `deploy_review` job calls `stop_review` -to clean up and stop the environment: - -```yaml -deploy_review: - stage: deploy - script: - - echo "Deploy a review app" - environment: - name: review/$CI_COMMIT_REF_NAME - url: https://$CI_ENVIRONMENT_SLUG.example.com - on_stop: stop_review - only: - - branches - except: - - master - -stop_review: - stage: deploy - variables: - GIT_STRATEGY: none - script: - - echo "Remove review app" - when: manual - environment: - name: review/$CI_COMMIT_REF_NAME - action: stop -``` - -Setting the [`GIT_STRATEGY`](yaml/README.md#git-strategy) to `none` is necessary in the -`stop_review` job so that the [GitLab Runner](https://docs.gitlab.com/runner/) won't -try to check out the code after the branch is deleted. - -When you have an environment that has a stop action defined (typically when -the environment describes a Review App), GitLab will automatically trigger a -stop action when the associated branch is deleted. The `stop_review` job must -be in the same `stage` as the `deploy_review` job in order for the environment -to automatically stop. - -You can read more in the [`.gitlab-ci.yml` reference](yaml/README.md#environmenton_stop). - -#### Environments auto-stop - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/20956) in GitLab 12.8. - -You can set a expiry time to environments and stop them automatically after a certain period. - -For example, consider the use of this feature with Review Apps environments. -When you set up Review Apps, sometimes they keep running for a long time -because some merge requests are left as open. An example for this situation is when the author of the merge -request is not actively working on it, due to priority changes or a different approach was decided on, and the merge requests was simply forgotten. -Idle environments waste resources, therefore they -should be terminated as soon as possible. - -To address this problem, you can specify an optional expiration date for -Review Apps environments. When the expiry time is reached, GitLab will automatically trigger a job -to stop the environment, eliminating the need of manually doing so. In case an environment is updated, the expiration is renewed -ensuring that only active merge requests keep running Review Apps. - -To enable this feature, you need to specify the [`environment:auto_stop_in`](yaml/README.md#environmentauto_stop_in) keyword in `.gitlab-ci.yml`. -You can specify a human-friendly date as the value, such as `1 hour and 30 minutes` or `1 day`. -`auto_stop_in` uses the same format of [`artifacts:expire_in` docs](yaml/README.md#artifactsexpire_in). - -##### Auto-stop example - -In the following example, there is a basic review app setup that creates a new environment -per merge request. The `review_app` job is triggered by every push and -creates or updates an environment named `review/your-branch-name`. -The environment keeps running until `stop_review_app` is executed: - -```yaml -review_app: - script: deploy-review-app - environment: - name: review/$CI_COMMIT_REF_NAME - on_stop: stop_review_app - auto_stop_in: 1 week - -stop_review_app: - script: stop-review-app - environment: - name: review/$CI_COMMIT_REF_NAME - action: stop - when: manual -``` - -As long as a merge request is active and keeps getting new commits, -the review app will not stop, so developers don't need to worry about -re-initiating review app. - -On the other hand, since `stop_review_app` is set to `auto_stop_in: 1 week`, -if a merge request becomes inactive for more than a week, -GitLab automatically triggers the `stop_review_app` job to stop the environment. - -You can also check the expiration date of environments through the GitLab UI. To do so, -go to **Operations > Environments > Environment**. You can see the auto-stop period -at the left-top section and a pin-mark button at the right-top section. This pin-mark -button can be used to prevent auto-stopping the environment. By clicking this button, the `auto_stop_in` setting is over-written -and the environment will be active until it's stopped manually. - - - -NOTE: **NOTE** -Due to the resource limitation, a background worker for stopping environments only -runs once every hour. This means environments will not be stopped at the exact -timestamp as the specified period, but will be stopped when the hourly cron worker -detects expired environments. - -#### Delete a stopped environment - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22629) in GitLab 12.9. - -You can delete [stopped environments](#stopping-an-environment) in one of two -ways: through the GitLab UI or through the API. - -##### Delete environments through the UI - -To view the list of **Stopped** environments, navigate to **Operations > Environments** -and click the **Stopped** tab. - -From there, you can click the **Delete** button directly, or you can click the -environment name to see its details and **Delete** it from there. - -You can also delete environments by viewing the details for a -stopped environment: - - 1. Navigate to **Operations > Environments**. - 1. Click on the name of an environment within the **Stopped** environments list. - 1. Click on the **Delete** button that appears at the top for all stopped environments. - 1. Finally, confirm your chosen environment in the modal that appears to delete it. - -##### Delete environments through the API - -Environments can also be deleted by using the [Environments API](../api/environments.md#delete-an-environment). - -### Grouping similar environments - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7015) in GitLab 8.14. - -As documented in [Configuring dynamic environments](#configuring-dynamic-environments), you can -prepend environment name with a word, followed by a `/`, and finally the branch -name, which is automatically defined by the `CI_COMMIT_REF_NAME` variable. - -In short, environments that are named like `type/foo` are all presented under the same -group, named `type`. - -In our [minimal example](#example-configuration), we named the environments `review/$CI_COMMIT_REF_NAME` -where `$CI_COMMIT_REF_NAME` is the branch name. Here is a snippet of the example: - -```yaml -deploy_review: - stage: deploy - script: - - echo "Deploy a review app" - environment: - name: review/$CI_COMMIT_REF_NAME -``` - -In this case, if you visit the **Environments** page and the branches -exist, you should see something like: - - - -### Monitoring environments - -If you have enabled [Prometheus for monitoring system and response metrics](../user/project/integrations/prometheus.md), -you can monitor the behavior of your app running in each environment. For the monitoring -dashboard to appear, you need to Configure Prometheus to collect at least one -[supported metric](../user/project/integrations/prometheus_library/index.md). - -NOTE: **Note:** -Since GitLab 9.2, all deployments to an environment are shown directly on the monitoring dashboard. - -Once configured, GitLab will attempt to retrieve [supported performance metrics](../user/project/integrations/prometheus_library/index.md) -for any environment that has had a successful deployment. If monitoring data was -successfully retrieved, a **Monitoring** button will appear for each environment. - - - -Clicking on the **Monitoring** button will display a new page showing up to the last -8 hours of performance data. It may take a minute or two for data to appear -after initial deployment. - -All deployments to an environment are shown directly on the monitoring dashboard, -which allows easy correlation between any changes in performance and new -versions of the app, all without leaving GitLab. - - - -#### Linking to external dashboard - -Add a [button to the Monitoring dashboard](../user/project/operations/linking_to_an_external_dashboard.md) linking directly to your existing external dashboards. - -#### Embedding metrics in GitLab Flavored Markdown - -Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab Flavored Markdown](../user/project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown) for more details. - -### Web terminals - -> Web terminals were added in GitLab 8.15 and are only available to project Maintainers and Owners. - -If you deploy to your environments with the help of a deployment service (for example, -the [Kubernetes integration](../user/project/clusters/index.md)), GitLab can open -a terminal session to your environment. - -This is a powerful feature that allows you to debug issues without leaving the comfort -of your web browser. To enable it, just follow the instructions given in the service integration -documentation. - -Once enabled, your environments will gain a "terminal" button: - - - -You can also access the terminal button from the page for a specific environment: - - - -Wherever you find it, clicking the button will take you to a separate page to -establish the terminal session: - - - -This works just like any other terminal. You'll be in the container created -by your deployment so you can: - -- Run shell commands and get responses in real time. -- Check the logs. -- Try out configuration or code tweaks etc. - -You can open multiple terminals to the same environment, they each get their own shell -session and even a multiplexer like `screen` or `tmux`. - -NOTE: **Note:** -Container-based deployments often lack basic tools (like an editor), and may -be stopped or restarted at any time. If this happens, you will lose all your -changes. Treat this as a debugging tool, not a comprehensive online IDE. - -### Check out deployments locally - -Since GitLab 8.13, a reference in the Git repository is saved for each deployment, so -knowing the state of your current environments is only a `git fetch` away. - -In your Git configuration, append the `[remote "<your-remote>"]` block with an extra -fetch line: - -```text -fetch = +refs/environments/*:refs/remotes/origin/environments/* -``` - -### Scoping environments with specs - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2112) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.4. -> - [Scoping for environment variables was moved to Core](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30779) to Core in GitLab 12.2. - -You can limit the environment scope of a variable by -defining which environments it can be available for. - -Wildcards can be used, and the default environment scope is `*`, which means -any jobs will have this variable, not matter if an environment is defined or -not. - -For example, if the environment scope is `production`, then only the jobs -having the environment `production` defined would have this specific variable. -Wildcards (`*`) can be used along with the environment name, therefore if the -environment scope is `review/*` then any jobs with environment names starting -with `review/` would have that particular variable. - -Some GitLab features can behave differently for each environment. -For example, you can -[create a secret variable to be injected only into a production environment](variables/README.md#limiting-environment-scopes-of-environment-variables). - -In most cases, these features use the _environment specs_ mechanism, which offers -an efficient way to implement scoping within each environment group. - -Let's say there are four environments: - -- `production` -- `staging` -- `review/feature-1` -- `review/feature-2` - -Each environment can be matched with the following environment spec: - -| Environment Spec | `production` | `staging` | `review/feature-1` | `review/feature-2` | -|:-----------------|:-------------|:----------|:-------------------|:-------------------| -| * | Matched | Matched | Matched | Matched | -| production | Matched | | | | -| staging | | Matched | | | -| review/* | | | Matched | Matched | -| review/feature-1 | | | Matched | | - -As you can see, you can use specific matching for selecting a particular environment, -and also use wildcard matching (`*`) for selecting a particular environment group, -such as [Review Apps](review_apps/index.md) (`review/*`). - -NOTE: **Note:** -The most _specific_ spec takes precedence over the other wildcard matching. -In this case, `review/feature-1` spec takes precedence over `review/*` and `*` specs. - -### Environments Dashboard **(PREMIUM)** - -See [Environments Dashboard](environments/environments_dashboard.md) for a summary of each -environment's operational health. - -## Limitations - -In the `environment: name`, you are limited to only the [predefined environment variables](variables/predefined_variables.md). -Re-using variables defined inside `script` as part of the environment name will not work. - -## Further reading - -Below are some links you may find interesting: - -- [The `.gitlab-ci.yml` definition of environments](yaml/README.md#environment) -- [A blog post on Deployments & Environments](https://about.gitlab.com/blog/2016/08/26/ci-deployment-and-environments/) -- [Review Apps - Use dynamic environments to deploy your code for every branch](review_apps/index.md) -- [Deploy Boards for your applications running on Kubernetes](../user/project/deploy_boards.md) **(PREMIUM)** - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](environments/index.md). diff --git a/doc/ci/environments/environments_dashboard.md b/doc/ci/environments/environments_dashboard.md index f82728cd587..4a72c0d6d62 100644 --- a/doc/ci/environments/environments_dashboard.md +++ b/doc/ci/environments/environments_dashboard.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/ci/environments/incremental_rollouts.md b/doc/ci/environments/incremental_rollouts.md index 0fa0af6a9fb..016a6ac7cad 100644 --- a/doc/ci/environments/incremental_rollouts.md +++ b/doc/ci/environments/incremental_rollouts.md @@ -46,7 +46,7 @@ application will be deployed to a single pod while the remaining 9 will present First we [define the template as manual](https://gitlab.com/gl-release/incremental-rollout-example/blob/master/.gitlab-ci.yml#L100-103): -```yml +```yaml .manual_rollout_template: &manual_rollout_template <<: *rollout_template stage: production @@ -55,7 +55,7 @@ First we [define the template as manual](https://gitlab.com/gl-release/increment Then we [define the rollout amount for each step](https://gitlab.com/gl-release/incremental-rollout-example/blob/master/.gitlab-ci.yml#L152-155): -```yml +```yaml rollout 10%: <<: *manual_rollout_template variables: @@ -86,7 +86,7 @@ countdown and then deploy. First we [define the template as timed](https://gitlab.com/gl-release/timed-rollout-example/blob/master/.gitlab-ci.yml#L86-89): -```yml +```yaml .timed_rollout_template: &timed_rollout_template <<: *rollout_template when: delayed @@ -95,13 +95,13 @@ First we [define the template as timed](https://gitlab.com/gl-release/timed-roll We can define the delay period using the `start_in` key: -```yml +```yaml start_in: 1 minutes ``` Then we [define the rollout amount for each step](https://gitlab.com/gl-release/timed-rollout-example/blob/master/.gitlab-ci.yml#L97-101): -```yml +```yaml timed rollout 30%: <<: *timed_rollout_template stage: timed rollout 30% diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md new file mode 100644 index 00000000000..b6ec30b5571 --- /dev/null +++ b/doc/ci/environments/index.md @@ -0,0 +1,991 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +disqus_identifier: 'https://docs.gitlab.com/ee/ci/environments.html' +--- + +# Environments and deployments + +> Introduced in GitLab 8.9. + +Environments allow control of the continuous deployment of your software, +all within GitLab. + +## Introduction + +There are many stages required in the software development process before the software is ready +for public consumption. + +For example: + +1. Develop your code. +1. Test your code. +1. Deploy your code into a testing or staging environment before you release it to the public. + +This helps find bugs in your software, and also in the deployment process as well. + +GitLab CI/CD is capable of not only testing or building your projects, but also +deploying them in your infrastructure, with the added benefit of giving you a +way to track your deployments. In other words, you will always know what is +currently being deployed or has been deployed on your servers. + +It's important to know that: + +- Environments are like tags for your CI jobs, describing where code gets deployed. +- Deployments are created when [jobs](../yaml/README.md#introduction) deploy versions of code to environments, + so every environment can have one or more deployments. + +GitLab: + +- Provides a full history of your deployments for each environment. +- Keeps track of your deployments, so you always know what is currently being deployed on your + servers. + +If you have a deployment service such as [Kubernetes](../../user/project/clusters/index.md) +associated with your project, you can use it to assist with your deployments, and +can even access a [web terminal](#web-terminals) for your environment from within GitLab! + +## Configuring environments + +Configuring environments involves: + +1. Understanding how [pipelines](../pipelines/index.md) work. +1. Defining environments in your project's [`.gitlab-ci.yml`](../yaml/README.md) file. +1. Creating a job configured to deploy your application. For example, a deploy job configured with [`environment`](../yaml/README.md#environment) to deploy your application to a [Kubernetes cluster](../../user/project/clusters/index.md). + +The rest of this section illustrates how to configure environments and deployments using +an example scenario. It assumes you have already: + +- Created a [project](../../gitlab-basics/create-project.md) in GitLab. +- Set up [a Runner](../runners/README.md). + +In the scenario: + +- We are developing an application. +- We want to run tests and build our app on all branches. +- Our default branch is `master`. +- We deploy the app only when a pipeline on `master` branch is run. + +### Defining environments + +Let's consider the following `.gitlab-ci.yml` example: + +```yaml +stages: + - test + - build + - deploy + +test: + stage: test + script: echo "Running tests" + +build: + stage: build + script: echo "Building the app" + +deploy_staging: + stage: deploy + script: + - echo "Deploy to staging server" + environment: + name: staging + url: https://staging.example.com + only: + - master +``` + +We have defined three [stages](../yaml/README.md#stages): + +- `test` +- `build` +- `deploy` + +The jobs assigned to these stages will run in this order. If any job fails, then +the pipeline fails and jobs that are assigned to the next stage won't run. + +In our case: + +- The `test` job will run first. +- Then the `build` job. +- Lastly the `deploy_staging` job. + +With this configuration, we: + +- Check that the tests pass. +- Ensure that our app is able to be built successfully. +- Lastly we deploy to the staging server. + +NOTE: **Note:** +The `environment` keyword defines where the app is deployed. +The environment `name` and `url` is exposed in various places +within GitLab. Each time a job that has an environment specified +succeeds, a deployment is recorded, along with the Git SHA, and environment name. + +CAUTION: **Caution**: +Some characters are not allowed in environment names. Use only letters, +numbers, spaces, and `-`, `_`, `/`, `{`, `}`, or `.`. Also, it must not start nor end with `/`. + +In summary, with the above `.gitlab-ci.yml` we have achieved the following: + +- All branches will run the `test` and `build` jobs. +- The `deploy_staging` job will run [only](../yaml/README.md#onlyexcept-basic) on the `master` + branch, which means all merge requests that are created from branches don't + get deployed to the staging server. +- When a merge request is merged, all jobs will run and the `deploy_staging` + job will deploy our code to a staging server while the deployment + will be recorded in an environment named `staging`. + +#### Environment variables and Runner + +Starting with GitLab 8.15, the environment name is exposed to the Runner in +two forms: + +- `$CI_ENVIRONMENT_NAME`. The name given in `.gitlab-ci.yml` (with any variables + expanded). +- `$CI_ENVIRONMENT_SLUG`. A "cleaned-up" version of the name, suitable for use in URLs, + DNS, etc. + +If you change the name of an existing environment, the: + +- `$CI_ENVIRONMENT_NAME` variable will be updated with the new environment name. +- `$CI_ENVIRONMENT_SLUG` variable will remain unchanged to prevent unintended side + effects. + +Starting with GitLab 9.3, the environment URL is exposed to the Runner via +`$CI_ENVIRONMENT_URL`. The URL is expanded from either: + +- `.gitlab-ci.yml`. +- The external URL from the environment if not defined in `.gitlab-ci.yml`. + +#### Set dynamic environment URLs after a job finishes + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. + +In a job script, you can specify a static [environment URL](#using-the-environment-url). +However, there may be times when you want a dynamic URL. For example, +if you deploy a Review App to an external hosting +service that generates a random URL per deployment, like `https://94dd65b.amazonaws.com/qa-lambda-1234567`, +you don't know the URL before the deployment script finishes. +If you want to use the environment URL in GitLab, you would have to update it manually. + +To address this problem, you can configure a deployment job to report back a set of +variables, including the URL that was dynamically-generated by the external service. +GitLab supports [dotenv](https://github.com/bkeepers/dotenv) file as the format, +and expands the `environment:url` value with variables defined in the dotenv file. + +To use this feature, specify the +[`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) keyword in `.gitlab-ci.yml`. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Set dynamic URLs after a job finished](https://youtu.be/70jDXtOf4Ig). + +##### Example of setting dynamic environment URLs + +The following example shows a Review App that creates a new environment +per merge request. The `review` job is triggered by every push, and +creates or updates an environment named `review/your-branch-name`. +The environment URL is set to `$DYNAMIC_ENVIRONMENT_URL`: + +```yaml +review: + script: + - DYNAMIC_ENVIRONMENT_URL=$(deploy-script) # In script, get the environment URL. + - echo "DYNAMIC_ENVIRONMENT_URL=$DYNAMIC_ENVIRONMENT_URL" >> deploy.env # Add the value to a dotenv file. + artifacts: + reports: + dotenv: deploy.env # Report back dotenv file to rails. + environment: + name: review/$CI_COMMIT_REF_SLUG + url: $DYNAMIC_ENVIRONMENT_URL # and set the variable produced in script to `environment:url` + on_stop: stop_review + +stop_review: + script: + - ./teardown-environment + when: manual + environment: + name: review/$CI_COMMIT_REF_SLUG + action: stop +``` + +As soon as the `review` job finishes, GitLab updates the `review/your-branch-name` +environment's URL. +It parses the report artifact `deploy.env`, registers a list of variables as runtime-created, +uses it for expanding `environment:url: $DYNAMIC_ENVIRONMENT_URL` and sets it to the environment URL. +You can also specify a static part of the URL at `environment:url:`, such as +`https://$DYNAMIC_ENVIRONMENT_URL`. If the value of `DYNAMIC_ENVIRONMENT_URL` is +`123.awesome.com`, the final result will be `https://123.awesome.com`. + +The assigned URL for the `review/your-branch-name` environment is visible in the UI. +[See where the environment URL is displayed](#using-the-environment-url). + +> **Notes:** +> +> - `stop_review` doesn't generate a dotenv report artifact, so it won't recognize the `DYNAMIC_ENVIRONMENT_URL` variable. Therefore you should not set `environment:url:` in the `stop_review` job. +> - If the environment URL is not valid (for example, the URL is malformed), the system doesn't update the environment URL. + +### Configuring manual deployments + +Adding `when: manual` to an automatically executed job's configuration converts it to +a job requiring manual action. + +To expand on the [previous example](#defining-environments), the following includes +another job that deploys our app to a production server and is +tracked by a `production` environment. + +The `.gitlab-ci.yml` file for this is as follows: + +```yaml +stages: + - test + - build + - deploy + +test: + stage: test + script: echo "Running tests" + +build: + stage: build + script: echo "Building the app" + +deploy_staging: + stage: deploy + script: + - echo "Deploy to staging server" + environment: + name: staging + url: https://staging.example.com + only: + - master + +deploy_prod: + stage: deploy + script: + - echo "Deploy to production server" + environment: + name: production + url: https://example.com + when: manual + only: + - master +``` + +The `when: manual` action: + +- Exposes a "play" button in GitLab's UI for that job. +- Means the `deploy_prod` job will only be triggered when the "play" button is clicked. + +You can find the "play" button in the pipelines, environments, deployments, and jobs views. + +| View | Screenshot | +|:----------------|:-------------------------------------------------------------------------------| +| Pipelines |  | +| Single pipeline |  | +| Environments |  | +| Deployments |  | +| Jobs |  | + +Clicking on the play button in any view will trigger the `deploy_prod` job, and the +deployment will be recorded as a new environment named `production`. + +NOTE: **Note:** +If your environment's name is `production` (all lowercase), +it will get recorded in [Value Stream Analytics](../../user/project/cycle_analytics.md). + +### Configuring dynamic environments + +Regular environments are good when deploying to "stable" environments like staging or production. + +However, for environments for branches other than `master`, dynamic environments +can be used. Dynamic environments make it possible to create environments on the fly by +declaring their names dynamically in `.gitlab-ci.yml`. + +Dynamic environments are a fundamental part of [Review apps](../review_apps/index.md). + +### Configuring incremental rollouts + +Learn how to release production changes to only a portion of your Kubernetes pods with +[incremental rollouts](../environments/incremental_rollouts.md). + +#### Allowed variables + +The `name` and `url` parameters for dynamic environments can use most available CI/CD variables, +including: + +- [Predefined environment variables](../variables/README.md#predefined-environment-variables) +- [Project and group variables](../variables/README.md#gitlab-cicd-environment-variables) +- [`.gitlab-ci.yml` variables](../yaml/README.md#variables) + +However, you cannot use variables defined: + +- Under `script`. +- On the Runner's side. + +There are also other variables that are unsupported in the context of `environment:name`. +For more information, see [Where variables can be used](../variables/where_variables_can_be_used.md). + +#### Example configuration + +GitLab Runner exposes various [environment variables](../variables/README.md) when a job runs, so +you can use them as environment names. + +In the following example, the job will deploy to all branches except `master`: + +```yaml +deploy_review: + stage: deploy + script: + - echo "Deploy a review app" + environment: + name: review/$CI_COMMIT_REF_NAME + url: https://$CI_ENVIRONMENT_SLUG.example.com + only: + - branches + except: + - master +``` + +In this example: + +- The job's name is `deploy_review` and it runs on the `deploy` stage. +- We set the `environment` with the `environment:name` as `review/$CI_COMMIT_REF_NAME`. + Since the [environment name](../yaml/README.md#environmentname) can contain slashes (`/`), we can + use this pattern to distinguish between dynamic and regular environments. +- We tell the job to run [`only`](../yaml/README.md#onlyexcept-basic) on branches, + [`except`](../yaml/README.md#onlyexcept-basic) `master`. + +For the value of: + +- `environment:name`, the first part is `review`, followed by a `/` and then `$CI_COMMIT_REF_NAME`, + which receives the value of the branch name. +- `environment:url`, we want a specific and distinct URL for each branch. `$CI_COMMIT_REF_NAME` + may contain a `/` or other characters that would be invalid in a domain name or URL, + so we use `$CI_ENVIRONMENT_SLUG` to guarantee that we get a valid URL. + + For example, given a `$CI_COMMIT_REF_NAME` of `100-Do-The-Thing`, the URL will be something + like `https://100-do-the-4f99a2.example.com`. Again, the way you set up + the web server to serve these requests is based on your setup. + + We have used `$CI_ENVIRONMENT_SLUG` here because it is guaranteed to be unique. If + you're using a workflow like [GitLab Flow](../../topics/gitlab_flow.md), collisions + are unlikely and you may prefer environment names to be more closely based on the + branch name. In that case, you could use `$CI_COMMIT_REF_NAME` in `environment:url` in + the example above: `https://$CI_COMMIT_REF_NAME.example.com`, which would give a URL + of `https://100-do-the-thing.example.com`. + +NOTE: **Note:** +You are not required to use the same prefix or only slashes (`/`) in the dynamic environments' +names. However, using this format will enable the [grouping similar environments](#grouping-similar-environments) +feature. + +### Configuring Kubernetes deployments + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6. + +If you are deploying to a [Kubernetes cluster](../../user/project/clusters/index.md) +associated with your project, you can configure these deployments from your +`gitlab-ci.yml` file. + +The following configuration options are supported: + +- [`namespace`](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) + +In the following example, the job will deploy your application to the +`production` Kubernetes namespace. + +```yaml +deploy: + stage: deploy + script: + - echo "Deploy to production server" + environment: + name: production + url: https://example.com + kubernetes: + namespace: production + only: + - master +``` + +When deploying to a Kubernetes cluster using GitLab's Kubernetes integration, +information about the cluster and namespace will be displayed above the job +trace on the deployment job page: + + + +NOTE: **Note:** +Kubernetes configuration is not supported for Kubernetes clusters +that are [managed by GitLab](../../user/project/clusters/index.md#gitlab-managed-clusters). +To follow progress on support for GitLab-managed clusters, see the +[relevant issue](https://gitlab.com/gitlab-org/gitlab/issues/38054). + +### Complete example + +The configuration in this section provides a full development workflow where your app is: + +- Tested. +- Built. +- Deployed as a Review App. +- Deployed to a staging server once the merge request is merged. +- Finally, able to be manually deployed to the production server. + +The following combines the previous configuration examples, including: + +- Defining [simple environments](#defining-environments) for testing, building, and deployment to staging. +- Adding [manual actions](#configuring-manual-deployments) for deployment to production. +- Creating [dynamic environments](#configuring-dynamic-environments) for deployments for reviewing. + +```yaml +stages: + - test + - build + - deploy + +test: + stage: test + script: echo "Running tests" + +build: + stage: build + script: echo "Building the app" + +deploy_review: + stage: deploy + script: + - echo "Deploy a review app" + environment: + name: review/$CI_COMMIT_REF_NAME + url: https://$CI_ENVIRONMENT_SLUG.example.com + only: + - branches + except: + - master + +deploy_staging: + stage: deploy + script: + - echo "Deploy to staging server" + environment: + name: staging + url: https://staging.example.com + only: + - master + +deploy_prod: + stage: deploy + script: + - echo "Deploy to production server" + environment: + name: production + url: https://example.com + when: manual + only: + - master +``` + +A more realistic example would also include copying files to a location where a +webserver (for example, NGINX) could then access and serve them. + +The example below will copy the `public` directory to `/srv/nginx/$CI_COMMIT_REF_SLUG/public`: + +```yaml +review_app: + stage: deploy + script: + - rsync -av --delete public /srv/nginx/$CI_COMMIT_REF_SLUG + environment: + name: review/$CI_COMMIT_REF_NAME + url: https://$CI_COMMIT_REF_SLUG.example.com +``` + +This example requires that NGINX and GitLab Runner are set up on the server this job will run on. + +NOTE: **Note:** +See the [limitations](#limitations) section for some edge cases regarding the naming of +your branches and Review Apps. + +The complete example provides the following workflow to developers: + +- Create a branch locally. +- Make changes and commit them. +- Push the branch to GitLab. +- Create a merge request. + +Behind the scenes, GitLab Runner will: + +- Pick up the changes and start running the jobs. +- Run the jobs sequentially as defined in `stages`: + - First, run the tests. + - If the tests succeed, build the app. + - If the build succeeds, the app is deployed to an environment with a name specific to the + branch. + +So now, every branch: + +- Gets its own environment. +- Is deployed to its own unique location, with the added benefit of: + - Having a [history of deployments](#viewing-deployment-history). + - Being able to [rollback changes](#retrying-and-rolling-back) if needed. + +For more information, see [Using the environment URL](#using-the-environment-url). + +### Protected environments + +Environments can be "protected", restricting access to them. + +For more information, see [Protected environments](protected_environments.md). + +## Working with environments + +Once environments are configured, GitLab provides many features for working with them, +as documented below. + +### Viewing environments and deployments + +A list of environments and deployment statuses is available on each project's **Operations > Environments** page. + +For example: + + + +This example shows: + +- The environment's name with a link to its deployments. +- The last deployment ID number and who performed it. +- The job ID of the last deployment with its respective job name. +- The commit information of the last deployment, such as who committed it, to what + branch, and the Git SHA of the commit. +- The exact time the last deployment was performed. +- A button that takes you to the URL that you defined under the `environment` keyword + in `.gitlab-ci.yml`. +- A button that re-deploys the latest deployment, meaning it runs the job + defined by the environment name for that specific commit. + +The information shown in the **Environments** page is limited to the latest +deployments, but an environment can have multiple deployments. + +> **Notes:** +> +> - While you can create environments manually in the web interface, we recommend +> that you define your environments in `.gitlab-ci.yml` first. They will +> be automatically created for you after the first deploy. +> - The environments page can only be viewed by users with [Reporter permission](../../user/permissions.md#project-members-permissions) +> and above. For more information on permissions, see the [permissions documentation](../../user/permissions.md). +> - Only deploys that happen after your `.gitlab-ci.yml` is properly configured +> will show up in the **Environment** and **Last deployment** lists. + +### Viewing deployment history + +GitLab keeps track of your deployments, so you: + +- Always know what is currently being deployed on your servers. +- Can have the full history of your deployments for every environment. + +Clicking on an environment shows the history of its deployments. Here's an example **Environments** page +with multiple deployments: + + + +This view is similar to the **Environments** page, but all deployments are shown. Also in this view +is a **Rollback** button. For more information, see [Retrying and rolling back](#retrying-and-rolling-back). + +### Retrying and rolling back + +If there is a problem with a deployment, you can retry it or roll it back. + +To retry or rollback a deployment: + +1. Navigate to **Operations > Environments**. +1. Click on the environment. +1. In the deployment history list for the environment, click the: + - **Retry** button next to the last deployment, to retry that deployment. + - **Rollback** button next to a previously successful deployment, to roll back to that deployment. + +#### What to expect with a rollback + +Pressing the **Rollback** button on a specific commit will trigger a _new_ deployment with its +own unique job ID. + +This means that you will see a new deployment that points to the commit you are rolling back to. + +NOTE: **Note:** +The defined deployment process in the job's `script` determines whether the rollback succeeds or not. + +### Using the environment URL + +The [environment URL](../yaml/README.md#environmenturl) is exposed in a few +places within GitLab: + +- In a merge request widget as a link: +  +- In the Environments view as a button: +  +- In the Deployments view as a button: +  + +You can see this information in a merge request itself if: + +- The merge request is eventually merged to the default branch (usually `master`). +- That branch also deploys to an environment (for example, `staging` or `production`). + +For example: + + + +#### Going from source files to public pages + +With GitLab's [Route Maps](../review_apps/index.md#route-maps) you can go directly +from source files to public pages in the environment set for Review Apps. + +### Stopping an environment + +Stopping an environment: + +- Moves it from the list of **Available** environments to the list of **Stopped** + environments on the [**Environments** page](#viewing-environments-and-deployments). +- Executes an [`on_stop` action](../yaml/README.md#environmenton_stop), if defined. + +This is often used when multiple developers are working on a project at the same time, +each of them pushing to their own branches, causing many dynamic environments to be created. + +NOTE: **Note:** +Starting with GitLab 8.14, dynamic environments are stopped automatically +when their associated branch is deleted. + +#### Automatically stopping an environment + +Environments can be stopped automatically using special configuration. + +Consider the following example where the `deploy_review` job calls `stop_review` +to clean up and stop the environment: + +```yaml +deploy_review: + stage: deploy + script: + - echo "Deploy a review app" + environment: + name: review/$CI_COMMIT_REF_NAME + url: https://$CI_ENVIRONMENT_SLUG.example.com + on_stop: stop_review + only: + - branches + except: + - master + +stop_review: + stage: deploy + variables: + GIT_STRATEGY: none + script: + - echo "Remove review app" + when: manual + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop +``` + +Setting the [`GIT_STRATEGY`](../yaml/README.md#git-strategy) to `none` is necessary in the +`stop_review` job so that the [GitLab Runner](https://docs.gitlab.com/runner/) won't +try to check out the code after the branch is deleted. + +When you have an environment that has a stop action defined (typically when +the environment describes a Review App), GitLab will automatically trigger a +stop action when the associated branch is deleted. The `stop_review` job must +be in the same `stage` as the `deploy_review` job in order for the environment +to automatically stop. + +You can read more in the [`.gitlab-ci.yml` reference](../yaml/README.md#environmenton_stop). + +#### Environments auto-stop + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/20956) in GitLab 12.8. + +You can set a expiry time to environments and stop them automatically after a certain period. + +For example, consider the use of this feature with Review Apps environments. +When you set up Review Apps, sometimes they keep running for a long time +because some merge requests are left as open. An example for this situation is when the author of the merge +request is not actively working on it, due to priority changes or a different approach was decided on, and the merge requests was simply forgotten. +Idle environments waste resources, therefore they +should be terminated as soon as possible. + +To address this problem, you can specify an optional expiration date for +Review Apps environments. When the expiry time is reached, GitLab will automatically trigger a job +to stop the environment, eliminating the need of manually doing so. In case an environment is updated, the expiration is renewed +ensuring that only active merge requests keep running Review Apps. + +To enable this feature, you need to specify the [`environment:auto_stop_in`](../yaml/README.md#environmentauto_stop_in) keyword in `.gitlab-ci.yml`. +You can specify a human-friendly date as the value, such as `1 hour and 30 minutes` or `1 day`. +`auto_stop_in` uses the same format of [`artifacts:expire_in` docs](../yaml/README.md#artifactsexpire_in). + +##### Auto-stop example + +In the following example, there is a basic review app setup that creates a new environment +per merge request. The `review_app` job is triggered by every push and +creates or updates an environment named `review/your-branch-name`. +The environment keeps running until `stop_review_app` is executed: + +```yaml +review_app: + script: deploy-review-app + environment: + name: review/$CI_COMMIT_REF_NAME + on_stop: stop_review_app + auto_stop_in: 1 week + +stop_review_app: + script: stop-review-app + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + when: manual +``` + +As long as a merge request is active and keeps getting new commits, +the review app will not stop, so developers don't need to worry about +re-initiating review app. + +On the other hand, since `stop_review_app` is set to `auto_stop_in: 1 week`, +if a merge request becomes inactive for more than a week, +GitLab automatically triggers the `stop_review_app` job to stop the environment. + +You can also check the expiration date of environments through the GitLab UI. To do so, +go to **Operations > Environments > Environment**. You can see the auto-stop period +at the left-top section and a pin-mark button at the right-top section. This pin-mark +button can be used to prevent auto-stopping the environment. By clicking this button, the `auto_stop_in` setting is over-written +and the environment will be active until it's stopped manually. + + + +NOTE: **NOTE** +Due to the resource limitation, a background worker for stopping environments only +runs once every hour. This means environments will not be stopped at the exact +timestamp as the specified period, but will be stopped when the hourly cron worker +detects expired environments. + +#### Delete a stopped environment + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22629) in GitLab 12.9. + +You can delete [stopped environments](#stopping-an-environment) in one of two +ways: through the GitLab UI or through the API. + +##### Delete environments through the UI + +To view the list of **Stopped** environments, navigate to **Operations > Environments** +and click the **Stopped** tab. + +From there, you can click the **Delete** button directly, or you can click the +environment name to see its details and **Delete** it from there. + +You can also delete environments by viewing the details for a +stopped environment: + + 1. Navigate to **Operations > Environments**. + 1. Click on the name of an environment within the **Stopped** environments list. + 1. Click on the **Delete** button that appears at the top for all stopped environments. + 1. Finally, confirm your chosen environment in the modal that appears to delete it. + +##### Delete environments through the API + +Environments can also be deleted by using the [Environments API](../../api/environments.md#delete-an-environment). + +### Grouping similar environments + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7015) in GitLab 8.14. + +As documented in [Configuring dynamic environments](#configuring-dynamic-environments), you can +prepend environment name with a word, followed by a `/`, and finally the branch +name, which is automatically defined by the `CI_COMMIT_REF_NAME` variable. + +In short, environments that are named like `type/foo` are all presented under the same +group, named `type`. + +In our [minimal example](#example-configuration), we named the environments `review/$CI_COMMIT_REF_NAME` +where `$CI_COMMIT_REF_NAME` is the branch name. Here is a snippet of the example: + +```yaml +deploy_review: + stage: deploy + script: + - echo "Deploy a review app" + environment: + name: review/$CI_COMMIT_REF_NAME +``` + +In this case, if you visit the **Environments** page and the branches +exist, you should see something like: + + + +### Monitoring environments + +If you have enabled [Prometheus for monitoring system and response metrics](../../user/project/integrations/prometheus.md), +you can monitor the behavior of your app running in each environment. For the monitoring +dashboard to appear, you need to Configure Prometheus to collect at least one +[supported metric](../../user/project/integrations/prometheus_library/index.md). + +NOTE: **Note:** +Since GitLab 9.2, all deployments to an environment are shown directly on the monitoring dashboard. + +Once configured, GitLab will attempt to retrieve [supported performance metrics](../../user/project/integrations/prometheus_library/index.md) +for any environment that has had a successful deployment. If monitoring data was +successfully retrieved, a **Monitoring** button will appear for each environment. + + + +Clicking on the **Monitoring** button will display a new page showing up to the last +8 hours of performance data. It may take a minute or two for data to appear +after initial deployment. + +All deployments to an environment are shown directly on the monitoring dashboard, +which allows easy correlation between any changes in performance and new +versions of the app, all without leaving GitLab. + + + +#### Linking to external dashboard + +Add a [button to the Monitoring dashboard](../../user/project/operations/linking_to_an_external_dashboard.md) linking directly to your existing external dashboards. + +#### Embedding metrics in GitLab Flavored Markdown + +Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab Flavored Markdown](../../user/project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown) for more details. + +### Web terminals + +> Web terminals were added in GitLab 8.15 and are only available to project Maintainers and Owners. + +If you deploy to your environments with the help of a deployment service (for example, +the [Kubernetes integration](../../user/project/clusters/index.md)), GitLab can open +a terminal session to your environment. + +This is a powerful feature that allows you to debug issues without leaving the comfort +of your web browser. To enable it, just follow the instructions given in the service integration +documentation. + +Once enabled, your environments will gain a "terminal" button: + + + +You can also access the terminal button from the page for a specific environment: + + + +Wherever you find it, clicking the button will take you to a separate page to +establish the terminal session: + + + +This works just like any other terminal. You'll be in the container created +by your deployment so you can: + +- Run shell commands and get responses in real time. +- Check the logs. +- Try out configuration or code tweaks etc. + +You can open multiple terminals to the same environment, they each get their own shell +session and even a multiplexer like `screen` or `tmux`. + +NOTE: **Note:** +Container-based deployments often lack basic tools (like an editor), and may +be stopped or restarted at any time. If this happens, you will lose all your +changes. Treat this as a debugging tool, not a comprehensive online IDE. + +### Check out deployments locally + +Since GitLab 8.13, a reference in the Git repository is saved for each deployment, so +knowing the state of your current environments is only a `git fetch` away. + +In your Git configuration, append the `[remote "<your-remote>"]` block with an extra +fetch line: + +```plaintext +fetch = +refs/environments/*:refs/remotes/origin/environments/* +``` + +### Scoping environments with specs + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2112) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.4. +> - [Scoping for environment variables was moved to Core](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30779) to Core in GitLab 12.2. + +You can limit the environment scope of a variable by +defining which environments it can be available for. + +Wildcards can be used, and the default environment scope is `*`, which means +any jobs will have this variable, not matter if an environment is defined or +not. + +For example, if the environment scope is `production`, then only the jobs +having the environment `production` defined would have this specific variable. +Wildcards (`*`) can be used along with the environment name, therefore if the +environment scope is `review/*` then any jobs with environment names starting +with `review/` would have that particular variable. + +Some GitLab features can behave differently for each environment. +For example, you can +[create a secret variable to be injected only into a production environment](../variables/README.md#limit-the-environment-scopes-of-environment-variables). + +In most cases, these features use the _environment specs_ mechanism, which offers +an efficient way to implement scoping within each environment group. + +Let's say there are four environments: + +- `production` +- `staging` +- `review/feature-1` +- `review/feature-2` + +Each environment can be matched with the following environment spec: + +| Environment Spec | `production` | `staging` | `review/feature-1` | `review/feature-2` | +|:-----------------|:-------------|:----------|:-------------------|:-------------------| +| * | Matched | Matched | Matched | Matched | +| production | Matched | | | | +| staging | | Matched | | | +| review/* | | | Matched | Matched | +| review/feature-1 | | | Matched | | + +As you can see, you can use specific matching for selecting a particular environment, +and also use wildcard matching (`*`) for selecting a particular environment group, +such as [Review Apps](../review_apps/index.md) (`review/*`). + +NOTE: **Note:** +The most _specific_ spec takes precedence over the other wildcard matching. +In this case, `review/feature-1` spec takes precedence over `review/*` and `*` specs. + +### Environments Dashboard **(PREMIUM)** + +See [Environments Dashboard](../environments/environments_dashboard.md) for a summary of each +environment's operational health. + +## Limitations + +In the `environment: name`, you are limited to only the [predefined environment variables](../variables/predefined_variables.md). +Re-using variables defined inside `script` as part of the environment name will not work. + +## Further reading + +Below are some links you may find interesting: + +- [The `.gitlab-ci.yml` definition of environments](../yaml/README.md#environment) +- [A blog post on Deployments & Environments](https://about.gitlab.com/blog/2016/08/26/ci-deployment-and-environments/) +- [Review Apps - Use dynamic environments to deploy your code for every branch](../review_apps/index.md) +- [Deploy Boards for your applications running on Kubernetes](../../user/project/deploy_boards.md) **(PREMIUM)** + +<!-- ## Troubleshooting + +Include any troubleshooting steps that you can foresee. If you know beforehand what issues +one might have when setting this up, or when something is changed, or on upgrading, it's +important to describe those, too. Think of things that may go wrong and include them here. +This is important to minimize requests for support, and to avoid doc comments with +questions that you know someone might ask. + +Each scenario can be a third-level heading, e.g. `### Getting error message X`. +If you have none to add when creating a doc, leave this section in place +but commented out to help encourage others to add to it in the future. --> diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index fb6cf2a710f..43ac42ea0c7 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts, howto --- @@ -8,7 +11,7 @@ type: concepts, howto ## Overview -[Environments](../environments.md) can be used for different reasons: +[Environments](../environments/index.md) can be used for different reasons: - Some of them are just for testing. - Others are for production. diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md index 2986afe8284..089babc8945 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md @@ -20,7 +20,7 @@ You will need to replace the `vault.example.com` URL below with the URL of your ## How it works -Each job has JSON Web Token (JWT) provided as environment variable named `CI_JOB_JWT`. This JWT can be used to authenticate with Vault using the [JWT Auth](https://www.vaultproject.io/docs/auth/jwt/#jwt-authentication) method. +Each job has JSON Web Token (JWT) provided as environment variable named `CI_JOB_JWT`. This JWT can be used to authenticate with Vault using the [JWT Auth](https://www.vaultproject.io/docs/auth/jwt#jwt-authentication) method. The JWT's payload looks like this: @@ -51,7 +51,7 @@ The JWT is encoded by using RS256 and signed with your GitLab instance's OpenID You can use this JWT and your instance's JWKS endpoint (`https://gitlab.example.com/-/jwks`) to authenticate with a Vault server that is configured to allow the JWT Authentication method for authentication. -When configuring roles in Vault, you can use [bound_claims](https://www.vaultproject.io/docs/auth/jwt/#bound-claims) to match against the JWT's claims and restrict which secrets each CI job has access to. +When configuring roles in Vault, you can use [bound_claims](https://www.vaultproject.io/docs/auth/jwt#bound-claims) to match against the JWT's claims and restrict which secrets each CI job has access to. To communicate with Vault, you can use either its CLI client or perform API requests (using `curl` or another client). @@ -70,7 +70,7 @@ $ vault kv get -field=password secret/myproject/production/db real-pa$$w0rd ``` -To configure your Vault server, start by enabling the [JWT Auth](https://www.vaultproject.io/docs/auth/jwt/) method: +To configure your Vault server, start by enabling the [JWT Auth](https://www.vaultproject.io/docs/auth/jwt) method: ```shell $ vault auth enable jwt diff --git a/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md b/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md index 848808f65ea..610220a6bff 100644 --- a/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md +++ b/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md @@ -269,7 +269,7 @@ we know we will need to access everything in the `built` folder, given by GitLab We'll also cache `node_modules` to avoid having to do a full re-pull of those dependencies: just pack them up in the cache. Here is the full `build` job: -```yml +```yaml build: stage: build script: @@ -296,7 +296,7 @@ the previous job. Lastly, by convention, we let GitLab CI/CD know this needs to the `build` job by giving it a `test` [stage](../../../ci/yaml/README.md#stages). Following the YAML structure, the `test` job should look like this: -```yml +```yaml test: stage: test script: @@ -318,7 +318,7 @@ we've added test artifacts and a test stage to our GitLab CI/CD pipeline using ` allowing us to run our tests by every push. Our entire `.gitlab-ci.yml` file should now look like this: -```yml +```yaml image: node:10 build: @@ -417,7 +417,7 @@ credentials, which will be the same two credentials (Key ID and Secret). It's a fully understand [IAM Best Practices in AWS](https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html). We need to add these credentials to GitLab: 1. Log into your AWS account and go to the [Security Credentials page](https://console.aws.amazon.com/iam/home#/security_credential) -1. Click the **Access Keys** section and **Create New Access Key**. Create the key and keep the id and secret around, you'll need them later +1. Click the **Access Keys** section and **Create New Access Key**. Create the key and keep the ID and secret around, you'll need them later  @@ -426,7 +426,7 @@ fully understand [IAM Best Practices in AWS](https://docs.aws.amazon.com/IAM/lat  -1. Add a key named `AWS_KEY_ID` and copy the key id from Step 2 into the **Value** textbox +1. Add a key named `AWS_KEY_ID` and copy the key ID from Step 2 into the **Value** textbox 1. Add a key named `AWS_KEY_SECRET` and copy the key secret from Step 2 into the **Value** textbox ### Deploy your game with GitLab CI/CD @@ -440,7 +440,7 @@ we add directives to ensure deployment `only` happens on pushes to `master`. Thi single branch still runs through CI, and only merging (or committing directly) to master will trigger the `deploy` job of our pipeline. Put these together to get the following: -```yml +```yaml deploy: stage: deploy variables: @@ -459,7 +459,7 @@ deploy: Be sure to update the region and S3 URL in that last script command to fit your setup. Our final configuration file `.gitlab-ci.yml` looks like: -```yml +```yaml image: node:10 build: @@ -516,7 +516,7 @@ Errors can be easily debugged through GitLab's build logs, and within minutes of you can see the changes live on your game. Setting up Continuous Integration and Continuous Deployment from the start with Dark Nova enables -rapid but stable development. We can easily test changes in a separate [environment](../../environments.md), +rapid but stable development. We can easily test changes in a separate [environment](../../environments/index.md), or multiple environments if needed. Balancing and updating a multiplayer game can be ongoing and tedious, but having faith in a stable deployment with GitLab CI/CD allows a lot of breathing room in quickly getting changes to players. diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index 07b054dd2cb..0331fa70f06 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -376,7 +376,7 @@ You might want to create another Envoy task to do that for you. We also create the `.env` file in the same path to set up our production environment variables for Laravel. These are persistent data and will be shared to every new release. -Now, we would need to deploy our app by running `envoy run deploy`, but it won't be necessary since GitLab can handle that for us with CI's [environments](../../environments.md), which will be described [later](#setting-up-gitlab-cicd) in this tutorial. +Now, we would need to deploy our app by running `envoy run deploy`, but it won't be necessary since GitLab can handle that for us with CI's [environments](../../environments/index.md), which will be described [later](#setting-up-gitlab-cicd) in this tutorial. Now it's time to commit [Envoy.blade.php](https://gitlab.com/mehranrasulian/laravel-sample/blob/master/Envoy.blade.php) and push it to the `master` branch. To keep things simple, we commit directly to `master`, without using [feature-branches](../../../topics/gitlab_flow.md#github-flow-as-a-simpler-alternative) since collaboration is beyond the scope of this tutorial. diff --git a/doc/ci/examples/test-and-deploy-python-application-to-heroku.md b/doc/ci/examples/test-and-deploy-python-application-to-heroku.md index 6d05c37390a..2c626cb458a 100644 --- a/doc/ci/examples/test-and-deploy-python-application-to-heroku.md +++ b/doc/ci/examples/test-and-deploy-python-application-to-heroku.md @@ -76,14 +76,21 @@ To build this project you also need to have [GitLab Runner](https://docs.gitlab. You can use public runners available on `gitlab.com` or you can register your own: ```shell +cat > /tmp/test-config.template.toml << EOF +[[runners]] +[runners.docker] +[[runners.docker.services]] +name = "postgres:latest" +EOF + gitlab-runner register \ --non-interactive \ --url "https://gitlab.com/" \ --registration-token "PROJECT_REGISTRATION_TOKEN" \ --description "python-3.5" \ --executor "docker" \ - --docker-image python:3.5 \ - --docker-services postgres:latest + --template-config /tmp/test-config.template.toml \ + --docker-image python:3.5 ``` With the command above, you create a runner that uses the [`python:3.5`](https://hub.docker.com/_/python) image and uses a [PostgreSQL](https://hub.docker.com/_/postgres) database. diff --git a/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md b/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md index e480f4565ce..5df407f19fe 100644 --- a/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md +++ b/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md @@ -64,7 +64,19 @@ You can do this through the [Heroku Dashboard](https://dashboard.heroku.com/). First install [Docker Engine](https://docs.docker.com/installation/). To build this project you also need to have [GitLab Runner](https://docs.gitlab.com/runner/). -You can use public runners available on `gitlab.com` or register your own: +You can use public runners available on `gitlab.com` or register your own. Start by +creating a template configuration file in order to pass complex configuration: + +```shell +cat > /tmp/test-config.template.toml << EOF +[[runners]] +[runners.docker] +[[runners.docker.services]] +name = "postgres:latest" +EOF +``` + +Finally, register the runner, passing the newly-created template configuration file: ```shell gitlab-runner register \ @@ -73,8 +85,8 @@ gitlab-runner register \ --registration-token "PROJECT_REGISTRATION_TOKEN" \ --description "ruby:2.6" \ --executor "docker" \ - --docker-image ruby:2.6 \ - --docker-services latest + --template-config /tmp/test-config.template.toml \ + --docker-image ruby:2.6 ``` With the command above, you create a Runner that uses the [`ruby:2.6`](https://hub.docker.com/_/ruby) image and uses a [PostgreSQL](https://hub.docker.com/_/postgres) database. diff --git a/doc/ci/examples/test-scala-application.md b/doc/ci/examples/test-scala-application.md index 92fb69286cf..1831417e48e 100644 --- a/doc/ci/examples/test-scala-application.md +++ b/doc/ci/examples/test-scala-application.md @@ -14,7 +14,7 @@ The following `.gitlab-ci.yml` should be added in the root of your repository to trigger CI: ``` yaml -image: java:8 +image: openjdk:8 stages: - test @@ -74,5 +74,5 @@ in the `.gitlab-ci.yml` file with your application's name. ## Heroku API key You can look up your Heroku API key in your -[account](https://dashboard.heroku.com/account). Add a [protected variable](../variables/README.md#protected-environment-variables) with +[account](https://dashboard.heroku.com/account). Add a [protected variable](../variables/README.md#protect-a-custom-variable) with this value in **Project ➔ Variables** with key `HEROKU_API_KEY`. diff --git a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md index 6d92c86c819..cd1ad923249 100644 --- a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md +++ b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md @@ -375,7 +375,7 @@ see if our latest code is running without errors. When we finish this edition, GitLab will start another build and show a **build running** badge. It is expected, after all we just configured GitLab CI/CD to do this for every push! But you may think "Why run build and tests for simple things like editing README.md?" and it is a good question. -For changes that don't affect your application, you can add the keyword [`[ci skip]`](../../yaml/README.md#skipping-jobs) +For changes that don't affect your application, you can add the keyword [`[ci skip]`](../../yaml/README.md#skip-pipeline) to commit message and the build related to that commit will be skipped. In the end, we finally got our pretty green build succeeded badge! By outputting the result on the diff --git a/doc/ci/img/metrics_reports.png b/doc/ci/img/metrics_reports.png Binary files differdeleted file mode 100644 index ffd9f6830a2..00000000000 --- a/doc/ci/img/metrics_reports.png +++ /dev/null diff --git a/doc/ci/img/metrics_reports_v13_0.png b/doc/ci/img/metrics_reports_v13_0.png Binary files differnew file mode 100644 index 00000000000..1597031db0b --- /dev/null +++ b/doc/ci/img/metrics_reports_v13_0.png diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index b16cde54b93..b7f1837d83e 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -76,6 +76,9 @@ to apply all the continuous methods (Continuous Integration, Delivery, and Deployment) to your software with no third-party application or integration needed. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Introduction to GitLab CI](https://www.youtube.com/watch?v=l5705U8s_nQ&t=397) from a recent GitLab meetup. + ### How GitLab CI/CD works To use GitLab CI/CD, all you need is an application codebase hosted in a @@ -105,7 +108,7 @@ The scripts are grouped into **jobs**, and together they compose a **pipeline**. A minimalist example of `.gitlab-ci.yml` file could contain: -```yml +```yaml before_script: - apt-get install rubygems ruby-dev -y @@ -133,7 +136,7 @@ displayed by GitLab:  At the end, if anything goes wrong, you can easily -[roll back](../environments.md#retrying-and-rolling-back) all the changes: +[roll back](../environments/index.md#retrying-and-rolling-back) all the changes:  @@ -204,7 +207,7 @@ according to each stage (Verify, Package, Release). With GitLab CI/CD you can also: - Easily set up your app's entire lifecycle with [Auto DevOps](../../topics/autodevops/index.md). -- Deploy your app to different [environments](../environments.md). +- Deploy your app to different [environments](../environments/index.md). - Install your own [GitLab Runner](https://docs.gitlab.com/runner/). - [Schedule pipelines](../pipelines/schedules.md). - Check for app vulnerabilities with [Security Test reports](../../user/application_security/index.md). **(ULTIMATE)** @@ -212,7 +215,7 @@ With GitLab CI/CD you can also: To see all CI/CD features, navigate back to the [CI/CD index](../README.md). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -Watch the video [GitLab CI Live Demo](https://www.youtube.com/watch?v=pBe4t1CD8Fc) with a deeper overview of GitLab CI/CD. +Watch the video [GitLab CI Live Demo](https://youtu.be/l5705U8s_nQ?t=369) with a deeper overview of GitLab CI/CD. ### Setting up GitLab CI/CD for the first time diff --git a/doc/ci/jenkins/index.md b/doc/ci/jenkins/index.md index 551e32ac816..c4346005138 100644 --- a/doc/ci/jenkins/index.md +++ b/doc/ci/jenkins/index.md @@ -13,6 +13,10 @@ First of all, our [Quick Start Guide](../quick_start/README.md) contains a good You may also be interested in [Auto DevOps](../../topics/autodevops/index.md) which can potentially be used to build, test, and deploy your applications with little to no configuration needed at all. +For an example of how to convert a Jenkins pipeline into a GitLab CI/CD pipeline, +or how to use Auto DevOps to test your code automatically, watch the +[Migrating from Jenkins to GitLab](https://www.youtube.com/watch?v=RlEVGOpYF5Y) video. + For advanced CI/CD teams, [templates](#templates) can enable the reuse of pipeline configurations. Otherwise, read on for important information that will help you get the ball rolling. Welcome @@ -40,6 +44,15 @@ things we have found that helps this: of the improvements that GitLab offers, and this requires (eventually) updating your implementation as part of the transition. +## JenkinsFile Wrapper + +We are building a [JenkinsFile Wrapper](https://gitlab.com/gitlab-org/jfr-container-builder/) which will allow +you to run a complete Jenkins instance inside of a GitLab job, including plugins. This can help ease the process +of transition, by letting you delay the migration of less urgent pipelines for a period of time. + +If you are interested, join our [public testing issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215675) to +If you are interested, you might be able to [help GitLab test the wrapper](https://gitlab.com/gitlab-org/gitlab/-/issues/215675). + ## Important product differences There are some high level differences between the products worth mentioning: @@ -113,7 +126,7 @@ There are some important differences in the way Runners work in comparison to ag If you are using `gitlab.com`, you can take advantage of our [shared Runner fleet](../../user/gitlab_com/index.md#shared-runners) to run jobs without provisioning your own Runners. We are investigating making them -[available for self-managed instances](https://gitlab.com/gitlab-org/customers-gitlab-com/issues/414) +[available for self-managed instances](https://gitlab.com/groups/gitlab-org/-/epics/835) as well. ## Groovy vs. YAML @@ -282,7 +295,7 @@ my_job: In GitLab, we use the [`variables` keyword](../yaml/README.md#variables) to define different variables at runtime. These can also be set up through the GitLab UI, under CI/CD settings. See also our [general documentation on variables](../variables/README.md), -including the section on [protected variables](../variables/README.md#protected-environment-variables) which can be used +including the section on [protected variables](../variables/README.md#protect-a-custom-variable) which can be used to limit access to certain variables to certain environments or runners: ```yaml diff --git a/doc/ci/junit_test_reports.md b/doc/ci/junit_test_reports.md index c79db6dfbea..a77044e849d 100644 --- a/doc/ci/junit_test_reports.md +++ b/doc/ci/junit_test_reports.md @@ -18,6 +18,8 @@ You can configure your job to use JUnit test reports, and GitLab will display a report on the merge request so that it's easier and faster to identify the failure without having to check the entire log. +If you don't use Merge Requests but still want to see the JUnit output without searching through job logs, the full [JUnit test reports](#viewing-junit-test-reports-on-gitlab) are available in the pipeline detail view. + ## Use cases Consider the following workflow: @@ -66,7 +68,7 @@ For a list of supported languages on JUnit tests, check the [Wikipedia article](https://en.wikipedia.org/wiki/JUnit#Ports). To enable the JUnit reports in merge requests, you need to add -[`artifacts:reports:junit`](yaml/README.md#artifactsreportsjunit) +[`artifacts:reports:junit`](pipelines/job_artifacts.md#artifactsreportsjunit) in `.gitlab-ci.yml`, and specify the path(s) of the generated test reports. In the following examples, the job in the `test` stage runs and GitLab @@ -235,15 +237,44 @@ You can view all the known test suites and click on each of these to see further details, including the cases that makeup the suite. Cases are ordered by status, with failed showing at the top, skipped next and successful cases last. +You can also retrieve the reports via the [GitLab API](../api/pipelines.md#get-a-pipelines-test-report). + ### Enabling the feature This feature comes with the `:junit_pipeline_view` feature flag disabled by default. This feature is disabled due to some performance issues with very large data sets. When [the performance is improved](https://gitlab.com/groups/gitlab-org/-/epics/2854), the feature will be enabled by default. -To enable this feature, ask a GitLab administrator with Rails console access to run the +To enable this feature, ask a GitLab administrator with [Rails console access](../administration/feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the following command: ```ruby Feature.enable(:junit_pipeline_view) ``` + +## Viewing JUnit screenshots on GitLab + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202114) in GitLab 13.0. + +If JUnit XML files contain an `attachment` tag, GitLab parses the attachment. + +Upload your screenshots as [artifacts](pipelines/job_artifacts.md#artifactsreportsjunit) to GitLab. The `attachment` tag **must** contain the absolute path to the screenshots you uploaded. + +```xml +<testcase time="1.00" name="Test"> + <system-out>[[ATTACHMENT|/absolute/path/to/some/file]]</system-out> +</testcase> +``` + +When [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/6061) is complete, the attached file will be visible on the pipeline details page. + +### Enabling the feature + +This feature comes with the `:junit_pipeline_screenshots_view` feature flag disabled by default. + +To enable this feature, ask a GitLab administrator with [Rails console access](../administration/feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the +following command: + +```ruby +Feature.enable(:junit_pipeline_screenshots_view) +``` diff --git a/doc/ci/large_repositories/index.md b/doc/ci/large_repositories/index.md index 6ac3fa2c92d..b4059fc252b 100644 --- a/doc/ci/large_repositories/index.md +++ b/doc/ci/large_repositories/index.md @@ -130,7 +130,7 @@ other using `docker` executor. ### `shell` executor example -Let's assume that you have the following [config.toml](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). +Let's assume that you have the following [`config.toml`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). ```toml concurrent = 4 @@ -155,7 +155,7 @@ This `config.toml`: ### `docker` executor example -Let's assume that you have the following [config.toml](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). +Let's assume that you have the following [`config.toml`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). ```toml concurrent = 4 @@ -216,7 +216,7 @@ but this brings administrative overhead as the `.gitlab-ci.yml` needs to be upda In such cases, it might be desirable to keep the `.gitlab-ci.yml` clone path agnostic, but make it a configuration of Runner. -We can extend our [config.toml](https://docs.gitlab.com/runner/configuration/advanced-configuration.html) +We can extend our [`config.toml`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html) with the following specification that will be used by Runner if `.gitlab-ci.yml` will not override it: ```toml diff --git a/doc/ci/merge_request_pipelines/img/merge_request_pipelines_doubled_MR_v12_09.png b/doc/ci/merge_request_pipelines/img/merge_request_pipelines_doubled_MR_v12_09.png Binary files differnew file mode 100644 index 00000000000..3e4c72b9279 --- /dev/null +++ b/doc/ci/merge_request_pipelines/img/merge_request_pipelines_doubled_MR_v12_09.png diff --git a/doc/ci/merge_request_pipelines/img/merge_request_pipelines_doubled_branch_v12_09.png b/doc/ci/merge_request_pipelines/img/merge_request_pipelines_doubled_branch_v12_09.png Binary files differnew file mode 100644 index 00000000000..dd70c3f8c20 --- /dev/null +++ b/doc/ci/merge_request_pipelines/img/merge_request_pipelines_doubled_branch_v12_09.png diff --git a/doc/ci/merge_request_pipelines/index.md b/doc/ci/merge_request_pipelines/index.md index b57340347d2..a724bf416b6 100644 --- a/doc/ci/merge_request_pipelines/index.md +++ b/doc/ci/merge_request_pipelines/index.md @@ -13,93 +13,47 @@ changes are pushed to a branch. If you want the pipeline to run jobs **only** when merge requests are created or updated, you can use *pipelines for merge requests*. -In the UI, these pipelines are labeled as `detached`. +In the UI, these pipelines are labeled as `detached`. Otherwise, these pipelines appear the same +as other pipelines. - - -A few notes: - -- Pipelines for merge requests are incompatible with - [CI/CD for external repositories](../ci_cd_for_external_repos/index.md). -- [Since GitLab 11.10](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25504), pipelines for merge requests require GitLab Runner 11.9. -- If you use this feature with [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md), - pipelines for merge requests take precedence over the other regular pipelines. - -## Configuring pipelines for merge requests - -To configure pipelines for merge requests, configure your [CI/CD configuration file](../yaml/README.md). -There are a few different ways to do this. - -### Enable pipelines for merge requests for all jobs - -The recommended method for enabling pipelines for merge requests for all jobs in -a pipeline is to use [`workflow:rules`](../yaml/README.md#workflowrules). - -In this example, the pipeline always runs for all merge requests, as well as for all changes -to the master branch: - -```yaml -workflow: - rules: - - if: $CI_MERGE_REQUEST_ID # Execute jobs in merge request context - - if: $CI_COMMIT_BRANCH == 'master' # Execute jobs when a new commit is pushed to master branch +Any user who has developer [permissions](../../user/permissions.md) +can run a pipeline for merge requests. -build: - stage: build - script: ./build - -test: - stage: test - script: ./test + -deploy: - stage: deploy - script: ./deploy -``` +NOTE: **Note**: +If you use this feature with [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md), +pipelines for merge requests take precedence over the other regular pipelines. -### Enable pipelines for merge requests for specific jobs +## Prerequisites -To enable pipelines for merge requests for specific jobs, you can use -[`rules`](../yaml/README.md#rules). +To enable pipelines for merge requests: -In the following example: +- You must have maintainer [permissions](../../user/permissions.md). +- Your repository must be a GitLab repository, not an + [external repository](../ci_cd_for_external_repos/index.md). +- [In GitLab 11.10 and later](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25504), + you must be using GitLab Runner 11.9. -- The `build` job runs for all changes to the `master` branch, as well as for all merge requests. -- The `test` job runs for all merge requests. -- The `deploy` job runs for all changes to the `master` branch, but does *not* run - for merge requests. +## Configuring pipelines for merge requests -```yaml -build: - stage: build - script: ./build - rules: - - if: $CI_COMMIT_BRANCH == 'master' # Execute jobs when a new commit is pushed to master branch - - if: $CI_MERGE_REQUEST_ID # Execute jobs in merge request context +To configure pipelines for merge requests you need to configure your [CI/CD configuration file](../yaml/README.md). +There are a few different ways to do this: -test: - stage: test - script: ./test - rules: - - if: $CI_MERGE_REQUEST_ID # Execute jobs in merge request context +### Use `rules` to run pipelines for merge requests -deploy: - stage: deploy - script: ./deploy - rules: - - if: $CI_COMMIT_BRANCH == 'master' # Execute jobs when a new commit is pushed to master branch -``` +When using `rules`, which is the preferred method, we recommend starting with one +of the [`workflow:rules` templates](../yaml/README.md#workflowrules-templates) to ensure +your basic configuration is correct. Instructions on how to do this, as well as how +to customize, are available at that link. ### Use `only` or `except` to run pipelines for merge requests -NOTE: **Note**: -The [`only` / `except`](../yaml/README.md#onlyexcept-basic) keywords are going to be deprecated -and you should not use them. - -To enable pipelines for merge requests, you can use `only / except`. When you use this method, -you have to specify `only: - merge_requests` for each job. +If you want to continue using `only/except`, this is possible but please review the drawbacks +below. -In this example, the pipeline contains a `test` job that is configured to run on merge requests. +When you use this method, you have to specify `only: - merge_requests` for each job. In this +example, the pipeline contains a `test` job that is configured to run on merge requests. The `build` and `deploy` jobs don't have the `only: - merge_requests` parameter, so they will not run on merge requests. @@ -245,14 +199,24 @@ to integrate your job with [GitLab Merge Request API](../../api/merge_requests.m You can find the list of available variables in [the reference sheet](../variables/predefined_variables.md). The variable names begin with the `CI_MERGE_REQUEST_` prefix. -<!-- ## Troubleshooting +## Troubleshooting + +### Two pipelines created when pushing to a merge request + +If you are experiencing duplicated pipelines when using `rules`, take a look at +the [key details when using `rules`](../yaml/README.md#key-details-when-using-rules), +which will help you get your starting configuration correct. + +If you are seeing two pipelines when using `only/except`, please see the caveats +related to using `only/except` above (or, consider moving to `rules`). + +### Two pipelines created when pushing an invalid CI configuration file + +Pushing to a branch with an invalid CI configuration file can trigger +the creation of two types of failed pipelines. One pipeline is a failed merge request +pipeline, and the other is a failed branch pipeline, but both are caused by the same +invalid configuration. -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +In rare cases, duplicate pipelines are created. -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/201845) for details. diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md index fb5c7830ac2..c35a5d0a07e 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md @@ -34,14 +34,18 @@ In these cases, the pipeline runs as a [pipeline for merge requests](../index.md and is labeled as `detached`. If these cases no longer exist, new pipelines will again run against the merged results. -## Requirements and limitations +Any user who has developer [permissions](../../../user/permissions.md) can run a +pipeline for merged results. -Pipelines for merged results have the following requirements and limitations: +## Prerequisites -- Pipelines for merged results require [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or newer. -- Forking/cross-repo workflows are not currently supported. To follow progress, +To enable pipelines for merge results: + +- You must have maintainer [permissions](../../../user/permissions.md). +- You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. +- You must not be forking or using cross-repo workflows. To follow progress, see [#11934](https://gitlab.com/gitlab-org/gitlab/issues/11934). -- This feature is not available for +- You must not be using [fast forward merges](../../../user/project/merge_requests/fast_forward_merge.md) yet. To follow progress, see [#58226](https://gitlab.com/gitlab-org/gitlab/-/issues/26996). @@ -93,7 +97,6 @@ canceled. Can be caused by some disabled feature flags. Please make sure that the following feature flags are enabled on your GitLab instance: -- `:ci_use_merge_request_ref` - `:merge_ref_auto_sync` To check and set these feature flag values, please ask an administrator to: @@ -107,14 +110,12 @@ To check and set these feature flag values, please ask an administrator to: 1. Check if the flags are enabled or not: ```ruby - Feature.enabled?(:ci_use_merge_request_ref) Feature.enabled?(:merge_ref_auto_sync) ``` 1. If needed, enable the feature flags: ```ruby - Feature.enable(:ci_use_merge_request_ref) Feature.enable(:merge_ref_auto_sync) ``` diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md index 641192afea1..d921b75aa44 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md @@ -30,6 +30,14 @@ If the pipeline for the merge request at the front of the train completes succes the changes are merged into the target branch, and the other pipelines continue to run. +To add a merge request to a merge train, you need [permissions](../../../../user/permissions.md) to push to the target branch. + +NOTE: **Note:** +Each merge train can run a maximum of **twenty** pipelines in parallel. +If more than twenty merge requests are added to the merge train, the merge requests +will be queued until a slot in the merge train is free. There is no limit to the +number of merge requests that can be queued. + ## Merge train example Three merge requests (`A`, `B` and `C`) are added to a merge train in order, which @@ -55,16 +63,13 @@ Watch this video for a demonstration on [how parallel execution of Merge Trains can prevent commits from breaking the default branch](https://www.youtube.com/watch?v=D4qCqXgZkHQ). -## Requirements and limitations +## Prerequisites -Merge trains have the following requirements and limitations: +To enable merge trains: -- Merge trains require [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or newer. -- GitLab 12.0 and later requires [Redis](https://redis.io/) 3.2 or higher. -- Each merge train can run a maximum of **twenty** pipelines in parallel. - If more than twenty merge requests are added to the merge train, the merge requests - will be queued until a slot in the merge train is free. There is no limit to the - number of merge requests that can be queued. +- You must have maintainer [permissions](../../../../user/permissions.md). +- You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. +- In GitLab 12.0 and later, you need [Redis](https://redis.io/) 3.2 or later. ## Enable merge trains diff --git a/doc/ci/metrics_reports.md b/doc/ci/metrics_reports.md index 871f8c55e83..f353aa2670f 100644 --- a/doc/ci/metrics_reports.md +++ b/doc/ci/metrics_reports.md @@ -12,7 +12,7 @@ GitLab provides a lot of great reporting tools for [merge requests](../user/proj You can configure your job to use custom Metrics Reports, and GitLab will display a report on the merge request so that it's easier and faster to identify changes without having to check the entire log. - + ## Use cases @@ -34,7 +34,7 @@ All values are considered strings and string compare is used to find differences ## How to set it up -Add a job that creates a [metrics report](yaml/README.md#artifactsreportsmetrics-premium) (default filename: `metrics.txt`). The file should conform to the [OpenMetrics](https://openmetrics.io/) format. +Add a job that creates a [metrics report](pipelines/job_artifacts.md#artifactsreportsmetrics-premium) (default filename: `metrics.txt`). The file should conform to the [OpenMetrics](https://openmetrics.io/) format. For example: diff --git a/doc/ci/parent_child_pipelines.md b/doc/ci/parent_child_pipelines.md index 2bc897901fa..e28adc2bc01 100644 --- a/doc/ci/parent_child_pipelines.md +++ b/doc/ci/parent_child_pipelines.md @@ -43,6 +43,9 @@ Child pipelines work well with other GitLab CI/CD features: All of this will work with the [`include:`](yaml/README.md#include) feature so you can compose the child pipeline configuration. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Parent-Child Pipelines feature demo](https://youtu.be/n8KpBSqZNbk). + ## Examples The simplest case is [triggering a child pipeline](yaml/README.md#trigger) using a @@ -136,6 +139,9 @@ your own script to generate a YAML file, which is then [used to trigger a child This technique can be very powerful in generating pipelines targeting content that changed or to build a matrix of targets and architectures. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Create child pipelines using dynamically generated configurations](https://youtu.be/nMdfus2JWHM). + In GitLab 12.9, the child pipeline could fail to be created in certain cases, causing the parent pipeline to fail. This is [resolved in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/209070). diff --git a/doc/ci/pipelines/img/pipelines_index.png b/doc/ci/pipelines/img/pipelines_index.png Binary files differdeleted file mode 100644 index e168e7e23df..00000000000 --- a/doc/ci/pipelines/img/pipelines_index.png +++ /dev/null diff --git a/doc/ci/pipelines/img/pipelines_index_v13_0.png b/doc/ci/pipelines/img/pipelines_index_v13_0.png Binary files differnew file mode 100644 index 00000000000..1dc5ea5d55c --- /dev/null +++ b/doc/ci/pipelines/img/pipelines_index_v13_0.png diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 27e968c0a40..731cb196eaa 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -74,7 +74,7 @@ You can also configure specific aspects of your pipelines through the GitLab UI. - [Pipeline settings](settings.md) for each project. - [Pipeline schedules](schedules.md). -- [Custom CI/CD variables](../variables/README.md#creating-a-custom-environment-variable). +- [Custom CI/CD variables](../variables/README.md#custom-environment-variables). ### View pipelines @@ -82,7 +82,7 @@ You can find the current and historical pipeline runs under your project's **CI/CD > Pipelines** page. You can also access pipelines for a merge request by navigating to its **Pipelines** tab. - + Clicking a pipeline will bring you to the **Pipeline Details** page and show the jobs that were run for that pipeline. From here you can cancel a running pipeline, @@ -93,6 +93,12 @@ latest pipeline for the last commit of a given branch is available at `/project/ Also, `/project/pipelines/latest` will redirect you to the latest pipeline for the last commit on the project's default branch. +[Starting in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/215367), +you can filter the pipeline list by: + +- Trigger author +- Branch name + ### Run a pipeline manually Pipelines can be manually executed, with predefined or manually-specified [variables](../variables/README.md). @@ -153,7 +159,7 @@ You can do this straight from the pipeline graph. Just click the play button to execute that particular job. For example, your pipeline might start automatically, but it requires manual action to -[deploy to production](../environments.md#configuring-manual-deployments). In the example below, the `production` +[deploy to production](../environments/index.md#configuring-manual-deployments). In the example below, the `production` stage has a job with a manual action.  @@ -216,7 +222,7 @@ In the example: Visually, it can be viewed as: -```text +```plaintext 0 1 2 3 4 5 6 7 AAAAAAA BBBBBBB @@ -225,7 +231,7 @@ Visually, it can be viewed as: The union of A, B, and C is (1, 4) and (6, 7). Therefore, the total running time is: -```text +```plaintext (4 - 1) + (7 - 6) => 4 ``` @@ -343,7 +349,7 @@ build ruby 2/3: stage: build script: - echo "ruby2" - + build ruby 3/3: stage: build script: @@ -367,10 +373,14 @@ evaluates the job names: `\d+[\s:\/\\]+\d+\s*`. When running manual jobs you can supply additional job specific variables. You can do this from the job page of the manual job you want to run with -additional variables. +additional variables. To access this page, click on the **name** of the manual job in +the pipeline view, *not* the play (**{play}**) button. -This is useful when you want to alter the execution of a job by using -environment variables. +This is useful when you want to alter the execution of a job that uses +[custom environment variables](../variables/README.md#custom-environment-variables). +Adding a variable name (key) and value here will override the value defined in +[the UI or `.gitlab-ci.yml`](../variables/README.md#custom-environment-variables), +for a single run of the manual job.  @@ -387,7 +397,7 @@ For example, if you start rolling out new code and: - Users do not experience trouble, GitLab can automatically complete the deployment from 0% to 100%. - Users experience trouble with the new code, you can stop the timed incremental rollout by canceling the pipeline - and [rolling](../environments.md#retrying-and-rolling-back) back to the last stable version. + and [rolling](../environments/index.md#retrying-and-rolling-back) back to the last stable version.  @@ -545,15 +555,3 @@ To illustrate its life cycle: even if the commit history of the `example` branch has been overwritten by force-push. 1. GitLab Runner fetches the persistent pipeline ref and gets source code from the checkout-SHA. 1. When the pipeline finished, its persistent ref is cleaned up in a background process. - -NOTE: **NOTE**: At this moment, this feature is on by default and can be manually disabled -by disabling `depend_on_persistent_pipeline_ref` feature flag. If you're interested in -manually disabling this behavior, please ask the administrator -to execute the following commands in rails console. - -```shell -> sudo gitlab-rails console # Login to Rails console of GitLab instance. -> project = Project.find_by_full_path('namespace/project-name') # Get the project instance. -> Feature.disable(:depend_on_persistent_pipeline_ref, project) # Disable the feature flag for specific project -> Feature.disable(:depend_on_persistent_pipeline_ref) # Disable the feature flag system-wide -``` diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index ed791ea9c4a..d4774016d9c 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -6,9 +6,9 @@ type: reference, howto # Job artifacts > - Introduced in GitLab 8.2 and GitLab Runner 0.7.0. -> - Starting with GitLab 8.4 and GitLab Runner 1.0, the artifacts archive format changed to `ZIP`, and it is now possible to browse its contents, with the added ability of downloading the files separately. +> - Starting with GitLab 8.4 and GitLab Runner 1.0, the artifacts archive format changed to `ZIP`, and it's now possible to browse its contents, with the added ability of downloading the files separately. > - In GitLab 8.17, builds were renamed to jobs. -> - The artifacts browser will be available only for new artifacts that are sent to GitLab using GitLab Runner version 1.0 and up. It will not be possible to browse old artifacts already uploaded to GitLab. +> - The artifacts browser will be available only for new artifacts that are sent to GitLab using GitLab Runner version 1.0 and up. It won't be possible to browse old artifacts already uploaded to GitLab. Job artifacts are a list of files and directories created by a job once it finishes. This feature is [enabled by default](../../administration/job_artifacts.md) in all @@ -34,7 +34,7 @@ pdf: expire_in: 1 week ``` -A job named `pdf` calls the `xelatex` command in order to build a pdf file from +A job named `pdf` calls the `xelatex` command in order to build a PDF file from the latex source file `mycv.tex`. We then define the `artifacts` paths which in turn are defined with the `paths` keyword. All paths to files and directories are relative to the repository that was cloned during the build. @@ -42,28 +42,229 @@ are relative to the repository that was cloned during the build. The artifacts will be uploaded when the job succeeds by default, but can be set to upload when the job fails, or always, if the [`artifacts:when`](../yaml/README.md#artifactswhen) parameter is used. These uploaded artifacts will be kept in GitLab for 1 week as defined -by the `expire_in` definition. You have the option to keep the artifacts from expiring +by the `expire_in` definition. You can keep the artifacts from expiring via the [web interface](#browsing-artifacts). If the expiry time is not defined, it defaults to the [instance wide setting](../../user/admin_area/settings/continuous_integration.md#default-artifacts-expiration-core-only). For more examples on artifacts, follow the [artifacts reference in `.gitlab-ci.yml`](../yaml/README.md#artifacts). +### `artifacts:reports` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20390) in GitLab 11.2. +> - Requires GitLab Runner 11.2 and above. + +The `artifacts:reports` keyword is used for collecting test reports, code quality +reports, and security reports from jobs. It also exposes these reports in GitLab's +UI (merge requests, pipeline views, and security dashboards). + +NOTE: **Note:** +The test reports are collected regardless of the job results (success or failure). +You can use [`artifacts:expire_in`](../yaml/README.md#artifactsexpire_in) to set up an expiration +date for their artifacts. + +NOTE: **Note:** +If you also want the ability to browse the report output files, include the +[`artifacts:paths`](../yaml/README.md#artifactspaths) keyword. + +#### `artifacts:reports:junit` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20390) in GitLab 11.2. +> - Requires GitLab Runner 11.2 and above. + +The `junit` report collects [JUnit XML files](https://www.ibm.com/support/knowledgecenter/en/SSQ2R2_14.1.0/com.ibm.rsar.analysis.codereview.cobol.doc/topics/cac_useresults_junit.html) +as artifacts. Although JUnit was originally developed in Java, there are many +[third party ports](https://en.wikipedia.org/wiki/JUnit#Ports) for other +languages like JavaScript, Python, Ruby, and so on. + +See [JUnit test reports](../junit_test_reports.md) for more details and examples. +Below is an example of collecting a JUnit XML file from Ruby's RSpec test tool: + +```yaml +rspec: + stage: test + script: + - bundle install + - rspec --format RspecJunitFormatter --out rspec.xml + artifacts: + reports: + junit: rspec.xml +``` + +The collected JUnit reports will be uploaded to GitLab as an artifact and will +be automatically shown in merge requests. + +NOTE: **Note:** +In case the JUnit tool you use exports to multiple XML files, you can specify +multiple test report paths within a single job and they will be automatically +concatenated into a single file. Use a filename pattern (`junit: rspec-*.xml`), +an array of filenames (`junit: [rspec-1.xml, rspec-2.xml, rspec-3.xml]`), or a +combination thereof (`junit: [rspec.xml, test-results/TEST-*.xml]`). + +#### `artifacts:reports:dotenv` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. +> - Requires GitLab Runner 11.5 and later. + +The `dotenv` report collects a set of environment variables as artifacts. + +The collected variables are registered as runtime-created variables of the job, +which is useful to [set dynamic environment URLs after a job finishes](../environments/index.md#set-dynamic-environment-urls-after-a-job-finishes). + +There are a couple of limitations on top of the [original dotenv rules](https://github.com/motdotla/dotenv#rules). + +- The variable key can contain only letters, digits and underscore ('_'). +- The size of the dotenv file must be smaller than 5 kilobytes. +- The number of variables must be less than 10. +- It does not support variable substitution in the dotenv file itself. +- It does not support empty lines and comments (`#`) in dotenv file. +- It does not support quote escape, spaces in a quote, a new line expansion in a quote, in dotenv file. + +#### `artifacts:reports:cobertura` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3708) in GitLab 12.9. +> - Requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 and above. + +The `cobertura` report collects [Cobertura coverage XML files](../../user/project/merge_requests/test_coverage_visualization.md). +The collected Cobertura coverage reports will be uploaded to GitLab as an artifact +and will be automatically shown in merge requests. + +Cobertura was originally developed for Java, but there are many +third party ports for other languages like JavaScript, Python, Ruby, and so on. + +#### `artifacts:reports:terraform` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207528) in GitLab 13.0. +> - Requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 and above. + +The `terraform` report obtains a Terraform `tfplan.json` file. The collected Terraform +plan report will be uploaded to GitLab as an artifact and will be automatically shown +in merge requests. + +#### `artifacts:reports:codequality` **(STARTER)** + +> - Introduced in GitLab 11.5. +> - Requires GitLab Runner 11.5 and above. + +The `codequality` report collects [CodeQuality issues](../../user/project/merge_requests/code_quality.md) +as artifacts. + +The collected Code Quality report will be uploaded to GitLab as an artifact and will +be summarized in merge requests. + +#### `artifacts:reports:sast` **(ULTIMATE)** + +> - Introduced in GitLab 11.5. +> - Requires GitLab Runner 11.5 and above. + +The `sast` report collects [SAST vulnerabilities](../../user/application_security/sast/index.md) +as artifacts. + +The collected SAST report will be uploaded to GitLab as an artifact and will be summarized +in the merge requests and pipeline view. It's also used to provide data for security +dashboards. + +#### `artifacts:reports:dependency_scanning` **(ULTIMATE)** + +> - Introduced in GitLab 11.5. +> - Requires GitLab Runner 11.5 and above. + +The `dependency_scanning` report collects [Dependency Scanning vulnerabilities](../../user/application_security/dependency_scanning/index.md) +as artifacts. + +The collected Dependency Scanning report will be uploaded to GitLab as an artifact and will +be summarized in the merge requests and pipeline view. It's also used to provide data for security +dashboards. + +#### `artifacts:reports:container_scanning` **(ULTIMATE)** + +> - Introduced in GitLab 11.5. +> - Requires GitLab Runner 11.5 and above. + +The `container_scanning` report collects [Container Scanning vulnerabilities](../../user/application_security/container_scanning/index.md) +as artifacts. + +The collected Container Scanning report will be uploaded to GitLab as an artifact and will +be summarized in the merge requests and pipeline view. It's also used to provide data for security +dashboards. + +#### `artifacts:reports:dast` **(ULTIMATE)** + +> - Introduced in GitLab 11.5. +> - Requires GitLab Runner 11.5 and above. + +The `dast` report collects [DAST vulnerabilities](../../user/application_security/dast/index.md) +as artifacts. + +The collected DAST report will be uploaded to GitLab as an artifact and will +be summarized in the merge requests and pipeline view. It's also used to provide data for security +dashboards. + +#### `artifacts:reports:license_management` **(ULTIMATE)** + +> - Introduced in GitLab 11.5. +> - Requires GitLab Runner 11.5 and above. + +CAUTION: **Warning:** +This artifact is still valid but is **deprecated** in favor of the +[artifacts:reports:license_scanning](../pipelines/job_artifacts.md#artifactsreportslicense_scanning-ultimate) +introduced in GitLab 12.8. + +The `license_management` report collects [Licenses](../../user/compliance/license_compliance/index.md) +as artifacts. + +The collected License Compliance report will be uploaded to GitLab as an artifact and will +be summarized in the merge requests and pipeline view. It's also used to provide data for security +dashboards. + +#### `artifacts:reports:license_scanning` **(ULTIMATE)** + +> - Introduced in GitLab 12.8. +> - Requires GitLab Runner 11.5 and above. + +The `license_scanning` report collects [Licenses](../../user/compliance/license_compliance/index.md) +as artifacts. + +The License Compliance report will be uploaded to GitLab as an artifact and will +be automatically shown in merge requests, pipeline view and provide data for security +dashboards. + +#### `artifacts:reports:performance` **(PREMIUM)** + +> - Introduced in GitLab 11.5. +> - Requires GitLab Runner 11.5 and above. + +The `performance` report collects [Performance metrics](../../user/project/merge_requests/browser_performance_testing.md) +as artifacts. + +The collected Performance report will be uploaded to GitLab as an artifact and will +be automatically shown in merge requests. + +#### `artifacts:reports:metrics` **(PREMIUM)** + +> Introduced in GitLab 11.10. + +The `metrics` report collects [Metrics](../metrics_reports.md) +as artifacts. + +The collected Metrics report will be uploaded to GitLab as an artifact and will +be automatically shown in merge requests. + ## Browsing artifacts -> - From GitLab 9.2, PDFs, images, videos and other formats can be previewed directly in the job artifacts browser without the need to download them. +> - From GitLab 9.2, PDFs, images, videos, and other formats can be previewed directly in the job artifacts browser without the need to download them. > - Introduced in [GitLab 10.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14399), HTML files in a public project can be previewed directly in a new tab without the need to download them when [GitLab Pages](../../administration/pages/index.md) is enabled. The same applies for textual formats (currently supported extensions: `.txt`, `.json`, and `.log`). > - Introduced in [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16675), artifacts in private projects can be previewed when [GitLab Pages access control](../../administration/pages/index.md#access-control) is enabled. After a job finishes, if you visit the job's specific page, there are three buttons. You can download the artifacts archive or browse its contents, whereas -the **Keep** button appears only if you have set an [expiry date](../yaml/README.md#artifactsexpire_in) to the +the **Keep** button appears only if you've set an [expiry date](../yaml/README.md#artifactsexpire_in) to the artifacts in case you changed your mind and want to keep them.  The archive browser shows the name and the actual file size of each file in the -archive. If your artifacts contained directories, then you are also able to +archive. If your artifacts contained directories, then you're also able to browse inside them. Below you can see what browsing looks like. In this case we have browsed inside @@ -75,20 +276,20 @@ one HTML file that you can view directly online when ## Downloading artifacts -If you need to download the whole archive, there are buttons in various places +If you need to download an artifact or the whole archive, there are buttons in various places in the GitLab UI to do this: 1. While on the pipelines page, you can see the download icon for each job's - artifacts archive in the right corner: + artifacts and archive in the right corner:  1. While on the **Jobs** page, you can see the download icon for each job's - artifacts archive in the right corner: + artifacts and archive in the right corner:  -1. While inside a specific job, you are presented with a download button +1. While inside a specific job, you're presented with a download button along with the one that browses the archive:  @@ -100,7 +301,7 @@ in the GitLab UI to do this: ## Downloading the latest artifacts -It is possible to download the latest artifacts of a job via a well known URL +It's possible to download the latest artifacts of a job via a well known URL so you can use it for scripting purposes. NOTE: **Note:** @@ -151,7 +352,7 @@ For example: https://gitlab.com/gitlab-org/gitlab/-/jobs/artifacts/master/browse?job=coverage ``` -There is also a URL to specific files, including html files that +There is also a URL to specific files, including HTML files that are shown in [GitLab Pages](../../administration/pages/index.md): ```plaintext diff --git a/doc/ci/pipelines/schedules.md b/doc/ci/pipelines/schedules.md index 0ca794c5411..0c0a613c628 100644 --- a/doc/ci/pipelines/schedules.md +++ b/doc/ci/pipelines/schedules.md @@ -21,6 +21,15 @@ Pipeline schedules can be used to also run [pipelines](index.md) at specific int In addition to using the GitLab UI, pipeline schedules can be maintained using the [Pipeline schedules API](../../api/pipeline_schedules.md). +## Prerequisites + +In order for a scheduled pipeline to be created successfully: + +- The schedule owner must have [permissions](../../user/permissions.md) to merge into the target branch. +- The pipeline configuration must be valid. + +Otherwise the pipeline is not created. + ## Configuring pipeline schedules To schedule a pipeline for project: diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 3221023f73a..0a859b5b68f 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -130,6 +130,16 @@ in the jobs table. A few examples of known coverage tools for a variety of languages can be found in the pipelines settings page. +### Download test coverage history + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209121) in GitLab 12.10. + +If you want to see the evolution of your project code coverage over time, +you can download a CSV file with this data. From your project: + +1. Go to **{chart}** **Project Analytics > Repository**. +1. Click **Download raw data (.csv)** + ### Removing color codes Some test coverage tools output with ANSI color codes that won't be @@ -162,6 +172,9 @@ This also determines the visibility of these related features: - Job artifacts - The [pipeline security dashboard](../../user/application_security/security_dashboard/index.md#pipeline-security) **(ULTIMATE)** +NOTE: **Note:** +Currently, job logs and artifacts are [not yet visible for guest users and non-project members](https://gitlab.com/gitlab-org/gitlab/-/issues/25649). + If **Public pipelines** is enabled (default): - For **public** projects, anyone can view the pipelines and related features. @@ -237,7 +250,7 @@ Depending on the status of your job, a badge can have the following values: You can access a pipeline status badge image using the following link: -```text +```plaintext https://example.gitlab.com/<namespace>/<project>/badges/<branch>/pipeline.svg ``` @@ -249,7 +262,7 @@ pipeline can have the test coverage percentage value defined. The test coverage badge can be accessed using following link: -```text +```plaintext https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg ``` @@ -268,7 +281,7 @@ Pipeline badges can be rendered in different styles by adding the `style=style_n #### Flat (default) -```text +```plaintext https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?style=flat ``` @@ -278,7 +291,7 @@ https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?st > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/30120) in GitLab 11.8. -```text +```plaintext https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?style=flat-square ``` diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index c3bdd524bff..ad0a4270f43 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -29,7 +32,7 @@ In the above example: ## How Review Apps work -A Review App is a mapping of a branch with an [environment](../environments.md). +A Review App is a mapping of a branch with an [environment](../environments/index.md). Access to the Review App is made available as a link on the [merge request](../../user/project/merge_requests.md) relevant to the branch. The following is an example of a merge request with an environment set dynamically. @@ -49,7 +52,7 @@ After adding Review Apps to your workflow, you follow the branched Git flow. Tha ## Configuring Review Apps -Review Apps are built on [dynamic environments](../environments.md#configuring-dynamic-environments), which allow you to dynamically create a new environment for each branch. +Review Apps are built on [dynamic environments](../environments/index.md#configuring-dynamic-environments), which allow you to dynamically create a new environment for each branch. The process of configuring Review Apps is as follows: @@ -58,7 +61,7 @@ The process of configuring Review Apps is as follows: 1. Set up a job in `.gitlab-ci.yml` that uses the [predefined CI environment variable](../variables/README.md) `${CI_COMMIT_REF_NAME}` to create dynamic environments and restrict it to run only on branches. Alternatively, you can get a YML template for this job by [enabling review apps](#enable-review-apps-button) for your project. -1. Optionally, set a job that [manually stops](../environments.md#stopping-an-environment) the Review Apps. +1. Optionally, set a job that [manually stops](../environments/index.md#stopping-an-environment) the Review Apps. ### Enable Review Apps button @@ -82,7 +85,7 @@ you can copy and paste into `.gitlab-ci.yml` as a starting point. To do so: ## Review Apps auto-stop -See how to [configure Review Apps environments to expire and auto-stop](../environments.md#environments-auto-stop) +See how to [configure Review Apps environments to expire and auto-stop](../environments/index.md#environments-auto-stop) after a given period of time. ## Review Apps examples @@ -100,7 +103,7 @@ See also the video [Demo: Cloud Native Development with GitLab](https://www.yout > Introduced in GitLab 8.17. In GitLab 11.5, the file links are available in the merge request widget. Route Maps allows you to go directly from source files -to public pages on the [environment](../environments.md) defined for +to public pages on the [environment](../environments/index.md) defined for Review Apps. Once set up, the review app link in the merge request @@ -205,7 +208,7 @@ if [route maps](#route-maps) are configured in the project.  -The provided script should be added to the `<head>` of you application and +The provided script should be added to the `<head>` of your application and consists of some project and merge request specific values. Here's what it looks like: @@ -267,7 +270,7 @@ to your review app. After determining the ID for the merge request to link to a visual review app, you can supply the ID by either: -- Hardcoding it in the script tag via the data attribute `data-merge-request-id` of the app. +- Hard-coding it in the script tag via the data attribute `data-merge-request-id` of the app. - Dynamically adding the `data-merge-request-id` value during the build of the app. - Supplying it manually through the visual review form in the app. @@ -278,7 +281,7 @@ can supply the ID by either: To enable visual reviews for private and internal projects, set the [`data-require-auth` variable](#configuring-visual-reviews) to `true`. When enabled, the user must enter a [personal access token](../../user/profile/personal_access_tokens.md) -with `read_api` scope before submitting feedback. +with `api` scope before submitting feedback. ### Using Visual Reviews @@ -301,4 +304,4 @@ automatically in the respective merge request. ## Limitations -Review App limitations are the same as [environments limitations](../environments.md#limitations). +Review App limitations are the same as [environments limitations](../environments/index.md#limitations). diff --git a/doc/ci/services/mysql.md b/doc/ci/services/mysql.md index e21b62e93cf..dcfd863709e 100644 --- a/doc/ci/services/mysql.md +++ b/doc/ci/services/mysql.md @@ -27,7 +27,7 @@ variables: NOTE: **Note:** The `MYSQL_DATABASE` and `MYSQL_ROOT_PASSWORD` variables can't be set in the GitLab UI. -To set them, assign them to a variable [in the UI](../variables/README.md#via-the-ui), +To set them, assign them to a variable [in the UI](../variables/README.md#create-a-custom-variable-in-the-ui), and then assign that variable to the `MYSQL_DATABASE` and `MYSQL_ROOT_PASSWORD` variables in your `.gitlab-ci.yml`. diff --git a/doc/ci/services/postgres.md b/doc/ci/services/postgres.md index cf34c28497e..2f92bd969ff 100644 --- a/doc/ci/services/postgres.md +++ b/doc/ci/services/postgres.md @@ -29,7 +29,7 @@ variables: NOTE: **Note:** The `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_HOST_AUTH_METHOD` variables can't be set in the GitLab UI. To set them, assign them to a variable -[in the UI](../variables/README.md#via-the-ui), and then assign that +[in the UI](../variables/README.md#create-a-custom-variable-in-the-ui), and then assign that variable to the `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_HOST_AUTH_METHOD` variables in your `.gitlab-ci.yml`. diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 99fbc2134a4..3e31a2169e2 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -4,149 +4,41 @@ type: reference # GitLab CI/CD environment variables -After a brief overview of the use of environment -variables, this document teaches you how to use GitLab CI/CD's -variables, presents the full reference for predefined variables, -and dives into more advanced applications. - -## Overview - -An environment variable is a dynamic-named value that can -affect the way running processes will behave on an operating +An environment variable is a dynamically-named value that can +affect the way running processes behave on an operating system. -They are part of the environment in which a process runs. +Environment variables are part of the environment in which a process runs. For example, a running process can query the value of the `TEMP` environment variable to discover a suitable location to store temporary files, or to define a `URL` for a database that can be reused in different scripts. -Variables are useful for customizing your jobs in GitLab -CI/CD's pipelines. Using variables means no hardcoded values. +Variables are useful for customizing your jobs in GitLab CI/CD. +When you use variables, you don't have to hard-code values. -### Predefined environment variables +## Predefined environment variables GitLab CI/CD has a [default set of predefined variables](predefined_variables.md) -which can be used without any specification needed. -You can call issues numbers, user names, branch names, +that you can use without any additional specification. +You can call issue numbers, user names, branch names, pipeline and commit IDs, and much more. -Predefined environment variables are the ones that GitLab -provides out of the box for the local environment of the Runner. - -GitLab reads the `.gitlab-ci.yml` file, sends the information -to the Runner (which runs the script commands), under which -the variables are exposed. - -For example, two jobs under the same pipeline can share the same -`CI_PIPELINE_ID` variable, but each one has its own `CI_JOB_ID` -variable. - -NOTE: **Note:** -Find here the full [**predefined variables reference table**](predefined_variables.md). - -### Custom environment variables - -When your use case requires a specific variable, you can -[set them up easily from the UI](#creating-a-custom-environment-variable) -or directly in the `.gitlab-ci.yml` file and reuse them as you wish. - -That can be very powerful as it can be used for scripting without -the need to specify the value itself. - -#### Types of variables - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/46806) in GitLab 11.11. - -There are two types of variables supported by GitLab: - -- [Variable type](#variable-type): The Runner will create an environment variable named the same as the - variable key and set its value to the variable value. -- [File type](#file-type): The Runner will write the variable value to a temporary file and set the - path to this file as the value of an environment variable, named the same as the variable key. - -##### Variable type - -Many tools (like [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) -and [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable)) -provide the ability to customise configuration using files by either providing the -file path as a command line argument or an environment variable. In the past, the -common pattern was to read the value of a CI variable, save it in a file, and then -use the newly created file in your script: - -```shell -# Read certificate stored in $KUBE_CA_PEM variable and save it in a new file -echo "$KUBE_CA_PEM" > "$(pwd)/kube.ca.pem" -# Pass the newly created file to kubectl -kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$(pwd)/kube.ca.pem" -``` - -There are [some predefined variables](#custom-variables-validated-by-gitlab) of this type, which may be further validated. They will appear when you add or update a variable. - -##### File type - -The example above can now be simplified by creating a "File" type variable, and using -it directly. For example, let's say we have the following variables: - - - -We can then call them from `.gitlab-ci.yml` like this: - -```shell -kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$KUBE_CA_PEM" -``` - -Variable types can be set via the [UI](#via-the-ui) or the [API](../../api/project_level_variables.md#create-variable), but not in `.gitlab-ci.yml`. - -#### Masked variables - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/13784) in GitLab 11.10 - -Variables can be created as masked variables. -This means that the value of the variable will be hidden in job logs, -though it must match certain requirements to do so: - -- The value must be in a single line. -- The value must only consist of characters from the Base64 alphabet (RFC4648). - - [In GitLab 12.2](https://gitlab.com/gitlab-org/gitlab-foss/issues/63043) - and newer, `@` and `:` are also valid values. -- The value must be at least 8 characters long. -- The value must not use variables. - -If the value does not meet the requirements above, then the CI variable will fail to save. -In order to save, either alter the value to meet the masking requirements -or disable **Masked** for the variable. - -#### Custom variables validated by GitLab - -Some variables are listed in the UI so you can choose them more quickly. -GitLab validates the values of these variables to ensure they are in the correct format. - -| Variable | Allowed Values | Introduced in | -|-------------------------|----------------------------------------------------|---------------| -| `AWS_ACCESS_KEY_ID` | 20 characters: letters, digits | 12.10 | -| `AWS_DEFAULT_REGION` | Any | 12.10 | -| `AWS_SECRET_ACCESS_KEY` | 40 characters: letters, digits, special characters | 12.10 | +Predefined environment variables are provided by GitLab +for the local environment of the Runner. -NOTE: **Note:** -When you store credentials, there are security implications. If you are using AWS keys, -for example, follow their [best practices](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html). +GitLab reads the `.gitlab-ci.yml` file and sends the information +to the Runner, where the variables are exposed. The Runner then runs the script commands. -## Getting started +### Use predefined environment variables -To get started with environment variables in the scope of GitLab -CI/CD, let's go over a few examples. +You can choose one of the existing predefined variables +to be output by the Runner. -### Using predefined environment variables +This example shows how to output a job's stage by using the predefined variable `CI_JOB_STAGE`. -To get started, choose one of the existing -[predefined variables](predefined_variables.md) -to be output by the Runner. For example, let's say that you want -a given job you're running through your script to output the -stage that job is running for. In your `.gitlab-ci.yml` file, -call the variable from your script according to the [syntaxes](#syntax-of-environment-variables-in-job-scripts) available. To -output the job stage, use the predefined variable `CI_JOB_STAGE`: +In your `.gitlab-ci.yml` file, call the variable from your script. Ensure +you use the correct [syntax](#syntax-of-environment-variables-in-job-scripts). ```yaml test_variable: @@ -155,14 +47,14 @@ test_variable: - echo $CI_JOB_STAGE ``` -For this case, the Runner will output the `stage` for the +In this case, the Runner outputs the `stage` for the job `test_variable`, which is `test`:  As another example, let's say you're using your own GitLab -instance you want to know what domain your GitLab Pages are -served under. You can easily call it with the predefined +instance and you want to know what domain your GitLab Pages are +served under. You can call it by using the predefined variable `$CI_PAGES_DOMAIN` in your script: ```yaml @@ -176,47 +68,54 @@ For GitLab.com users, the output will be `gitlab.io`. For your private instance, the output will be whatever your sysadmin has defined. -### Creating a custom environment variable +## Custom environment variables -Assume you have something you want to repeat through your scripts -in GitLab CI/CD's configuration file. To keep this example simple, -let's say you want to output `HELLO WORLD` for a `TEST` variable. +When you need a specific custom environment variable, you can +[set it up in the UI](#create-a-custom-variable-in-the-ui), in [the API](../../api/project_level_variables.md), +or directly [in the `.gitlab-ci.yml` file](#create-a-custom-variable-in-gitlab-ciyml). -You can either set the variable directly in the `.gitlab-ci.yml` -file or through the UI. +The variables are used by the Runner any time the pipeline runs. +You can also [override variable values manually for a specific pipeline](../pipelines/index.md#specifying-variables-when-running-manual-jobs). -NOTE: **Note:** -It is possible to [specify variables when running manual jobs](../pipelines/index.md#specifying-variables-when-running-manual-jobs). +There are two types of variables: **Variable** and **File**. You cannot set types in +the `.gitlab-ci.yml` file, but you can set them in the UI and API. -#### Via `.gitlab-ci.yml` +### Create a custom variable in `.gitlab-ci.yml` -To create a new custom `env_var` variable via [`.gitlab-ci.yml`](../yaml/README.md#variables), define their variable/value pair under -`variables`: +To create a custom `env_var` variable in the [`.gitlab-ci.yml`](../yaml/README.md#variables) file, +define the variable/value pair under `variables`: ```yaml variables: TEST: "HELLO WORLD" ``` -For a deeper look into them, see [`.gitlab-ci.yml` defined variables](#gitlab-ciyml-defined-variables). +You can then call its value in your script: + +```yaml + script: + - echo "$TEST" +``` + +For more details, see [`.gitlab-ci.yml` defined variables](#gitlab-ciyml-defined-variables). -#### Via the UI +### Create a custom variable in the UI From within the UI, you can add or update custom environment variables: 1. Go to your project's **Settings > CI/CD** and expand the **Variables** section. -1. Click the **Add variable** button. In the **Add variable** modal, fill in the details: +1. Click the **Add Variable** button. In the **Add variable** modal, fill in the details: - **Key**: Must be one line, with no spaces, using only letters, numbers, `-` or `_`. - **Value**: No limitations. - **Type**: `File` or `Variable`. - **Environment scope**: `All`, or specific environments. - **Protect variable** (Optional): If selected, the variable will only be available in pipelines that run on protected branches or tags. - - **Mask variable** (Optional): If selected, the variable's **Value** will be masked in job logs. The variable will fail to save if the value does not meet the [masking requirements](#masked-variables). + - **Mask variable** (Optional): If selected, the variable's **Value** will be masked in job logs. The variable fails to save if the value does not meet the [masking requirements](#masked-variable-requirements). -After a variable is created, you can update any of the details by clicking on the **{pencil}** **Edit** button. +After a variable is created, you can update any of the details by clicking the **{pencil}** **Edit** button. -Once you've set the variables, call them from the `.gitlab-ci.yml` file: +After you set a variable, call it from the `.gitlab-ci.yml` file: ```yaml test_variable: @@ -232,7 +131,110 @@ The output will be:  -### Syntax of environment variables in job scripts +### Custom environment variables of type Variable + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/46806) in GitLab 11.11. + +For variables with the type **Variable**, the Runner creates an environment variable +that uses the key for the name and the value for the value. + +There are [some predefined variables](#custom-variables-validated-by-gitlab) of this type, +which may be further validated. They appear when you add or update a variable in the UI. + +### Custom environment variables of type File + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/46806) in GitLab 11.11. + +For variables with the type **File**, the Runner creates an environment variable that uses the key for the name. +For the value, the Runner writes the variable value to a temporary file and uses this path. + +You can use tools like [the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) +and [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable) +to customize your configuration by using **File** type variables. + +In the past, a common pattern was to read the value of a CI variable, save it in a file, and then +use the newly created file in your script: + +```shell +# Read certificate stored in $KUBE_CA_PEM variable and save it in a new file +echo "$KUBE_CA_PEM" > "$(pwd)/kube.ca.pem" +# Pass the newly created file to kubectl +kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$(pwd)/kube.ca.pem" +``` + +Instead of this, you can use a **File** type variable. For example, if you have the following variables: + +- A variable of type **Variable**: `KUBE_URL` with the value `https://example.com`. +- A variable of type **File**: `KUBE_CA_PEM` with a certificate as the value. + +You can call them from `.gitlab-ci.yml`, like this: + +```shell +kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$KUBE_CA_PEM" +``` + +### Mask a custom variable + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/13784) in GitLab 11.10 + +Variables can be masked so that the value of the variable will be hidden in job logs. + +To mask a variable: + +1. Go to **Settings > CI/CD**. +1. Expand the **Variables** section. +1. Next to the variable you want to protect, click **Edit**. +1. Select the **Mask variable** check box. +1. Click **Update variable**. + +#### Masked variable requirements + +The value of the variable must: + +- Be in a single line. +- Be at least 8 characters long. +- Not be a predefined or custom environment variable. +- Consist only of characters from the Base64 alphabet (RFC4648). + [In GitLab 12.2](https://gitlab.com/gitlab-org/gitlab-foss/issues/63043) + and newer, `@` and `:` are also valid values. + +You can't mask variables that don't meet these requirements. + +### Protect a custom variable + +> Introduced in GitLab 9.3. + +Variables can be protected. When a variable is +protected, it is securely passed to pipelines running on +[protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md) only. The other pipelines do not get +the protected variable. + +To protect a variable: + +1. Go to **Settings > CI/CD**. +1. Expand the **Variables** section. +1. Next to the variable you want to protect, click **Edit**. +1. Select the **Protect variable** check box. +1. Click **Update variable**. + +The variable is available for all subsequent pipelines. + +### Custom variables validated by GitLab + +Some variables are listed in the UI so you can choose them more quickly. +GitLab validates the values of these variables to ensure they are in the correct format. + +| Variable | Allowed Values | Introduced in | +|-------------------------|----------------------------------------------------|---------------| +| `AWS_ACCESS_KEY_ID` | 20 characters: letters, digits | 12.10 | +| `AWS_DEFAULT_REGION` | Any | 12.10 | +| `AWS_SECRET_ACCESS_KEY` | 40 characters: letters, digits, special characters | 12.10 | + +NOTE: **Note:** +When you store credentials, there are security implications. If you are using AWS keys, +for example, follow their [best practices](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html). + +## Syntax of environment variables in job scripts All variables are set as environment variables in the build environment, and they are accessible with normal methods that are used to access such variables. @@ -329,14 +331,14 @@ export GITLAB_USER_EMAIL="user@example.com" export GITLAB_USER_ID="42" ``` -### `.gitlab-ci.yml` defined variables +## `.gitlab-ci.yml` defined variables NOTE: **Note:** This feature requires GitLab Runner 0.5.0 or higher and GitLab 7.14 or higher. -GitLab CI/CD allows you to add to `.gitlab-ci.yml` variables that are set in the -build environment. The variables are hence saved in the repository, and they -are meant to store non-sensitive project configuration. For example, `RAILS_ENV` or +You can add variables that are set in the build environment to `.gitlab-ci.yml`. +These variables are saved in the repository, and they +are meant to store non-sensitive project configuration, like `RAILS_ENV` or `DATABASE_URL`. For example, if you set the variable below globally (not inside a job), it will @@ -348,7 +350,7 @@ variables: ``` The YAML-defined variables are also set to all created -[service containers](../docker/using_docker_images.md), thus allowing to fine +[service containers](../docker/using_docker_images.md), so that you can fine tune them. Variables can be defined at a global level, but also at a job level. To turn off @@ -369,11 +371,11 @@ script: - 'eval $LS_CMD' # will execute 'ls -al $TMP_DIR' ``` -### Group-level environment variables +## Group-level environment variables > Introduced in GitLab 9.4. -GitLab CI/CD allows you to define per-project or per-group variables +You can define per-project or per-group variables that are set in the pipeline environment. Group-level variables are stored out of the repository (not in `.gitlab-ci.yml`) and are securely passed to GitLab Runner making them available during a pipeline run. It's the **recommended method** to @@ -382,7 +384,7 @@ use for storing things like passwords, SSH keys, and credentials. Group-level variables can be added by: 1. Navigating to your group's **Settings > CI/CD** page. -1. Inputing variable types, keys, and values in the **Variables** section. +1. Inputting variable types, keys, and values in the **Variables** section. Any variables of [subgroups](../../user/group/subgroups/index.md) will be inherited recursively. Once you set them, they will be available for all subsequent pipelines. Any group-level user defined variables can be viewed in projects by: @@ -392,6 +394,73 @@ Once you set them, they will be available for all subsequent pipelines. Any grou  +### Inherit environment variables + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22638) in GitLab 13.0. +> - It's deployed behind a feature flag (`ci_dependency_variables`), disabled by default. + +You can inherit environment variables from dependent jobs. + +This feature makes use of the [`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) report feature. + +Example with [`dependencies`](../yaml/README.md#dependencies) keyword. + +```yaml +build: + stage: build + script: + - echo "BUILD_VERSION=hello" >> build.env + artifacts: + reports: + dotenv: build.env + +deploy: + stage: deploy + script: + - echo $BUILD_VERSION # => hello + dependencies: + - build +``` + +Example with the [`needs`](../yaml/README.md#artifact-downloads-with-needs) keyword: + +```yaml +build: + stage: build + script: + - echo "BUILD_VERSION=hello" >> build.env + artifacts: + reports: + dotenv: build.env + +deploy: + stage: deploy + script: + - echo $BUILD_VERSION # => hello + needs: + - job: build + artifacts: true +``` + +### Enable inherited environment variables **(CORE ONLY)** + +The Inherited Environment Variables feature is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:ci_dependency_variables) +``` + +To disable it: + +```ruby +Feature.disable(:ci_dependency_variables) +``` + ## Priority of environment variables Variables of different types can take precedence over other @@ -400,8 +469,9 @@ variables, depending on where they are defined. The order of precedence for variables is (from highest to lowest): 1. [Trigger variables](../triggers/README.md#making-use-of-trigger-variables) or [scheduled pipeline variables](../pipelines/schedules.md#using-variables). -1. Project-level [variables](#creating-a-custom-environment-variable) or [protected variables](#protected-environment-variables). -1. Group-level [variables](#group-level-environment-variables) or [protected variables](#protected-environment-variables). +1. Project-level [variables](#custom-environment-variables) or [protected variables](#protect-a-custom-variable). +1. Group-level [variables](#group-level-environment-variables) or [protected variables](#protect-a-custom-variable). +1. [Inherited environment variables](#inherit-environment-variables). 1. YAML-defined [job-level variables](../yaml/README.md#variables). 1. YAML-defined [global variables](../yaml/README.md#variables). 1. [Deployment variables](#deployment-environment-variables). @@ -426,27 +496,12 @@ Click [here](where_variables_can_be_used.md) for a section that describes where ## Advanced use -### Protected environment variables - -> Introduced in GitLab 9.3. - -Variables can be protected. Whenever a variable is -protected, it would only be securely passed to pipelines running on the -[protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). The other pipelines would not get any -protected variables. - -Protected variables can be added by going to your project's -**Settings > CI/CD**, then finding the section called -**Variables**, and check "Protected". - -Once you set them, they will be available for all subsequent pipelines. - -### Limiting environment scopes of environment variables +### Limit the environment scopes of environment variables You can limit the environment scope of a variable by -[defining which environments](../environments.md) it can be available for. +[defining which environments](../environments/index.md) it can be available for. -To learn more about scoping environments, see [Scoping environments with specs](../environments.md#scoping-environments-with-specs). +To learn more about scoping environments, see [Scoping environments with specs](../environments/index.md#scoping-environments-with-specs). ### Deployment environment variables @@ -455,7 +510,7 @@ To learn more about scoping environments, see [Scoping environments with specs]( [Integrations](../../user/project/integrations/overview.md) that are responsible for deployment configuration may define their own variables that are set in the build environment. These variables are only defined for -[deployment jobs](../environments.md). Please consult the documentation of +[deployment jobs](../environments/index.md). Please consult the documentation of the integrations that you are using to learn which variables they define. An example integration that defines deployment variables is the @@ -478,22 +533,23 @@ CAUTION: **Caution:** Variables with multiline values are not currently supported due to limitations with the current Auto DevOps scripting environment. -### Environment variables triggered manually +### Override a variable by manually running a pipeline > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/44059) in GitLab 10.8. -[Manually triggered pipelines](../pipelines/index.md#run-a-pipeline-manually) allow you to override the value of a current variable. +You can override the value of a current variable by +[running a pipeline manually](../pipelines/index.md#run-a-pipeline-manually). + +For instance, suppose you added a custom variable named `$TEST` +and you want to override it in a manual pipeline. -For instance, suppose you added a -[custom variable `$TEST`](#creating-a-custom-environment-variable) -as exemplified above and you want to override it in a manual pipeline. Navigate to your project's **CI/CD > Pipelines** and click **Run pipeline**. -Choose the branch you want to run the pipeline for, then add a new variable through the UI: +Choose the branch you want to run the pipeline for, then add a variable and its value in the UI:  -The Runner will override the value previously set and use the custom -value you set for this specific pipeline: +The Runner overrides the value previously set and uses the custom +value for this specific pipeline.  @@ -502,10 +558,10 @@ value you set for this specific pipeline: > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/37397) in GitLab 10.7 for [the `only` and `except` CI keywords](../yaml/README.md#onlyexcept-advanced) > - [Expanded](https://gitlab.com/gitlab-org/gitlab/issues/27863) in GitLab 12.3 with [the `rules` keyword](../yaml/README.md#rules) -Variable expressions can be used to limit what jobs are going to be created -within a pipeline after pushing changes to GitLab. +Use variable expressions to limit which jobs are created +within a pipeline after changes are pushed to GitLab. -In `.gitlab-ci.yml`, they work with both +In `.gitlab-ci.yml`, variable expressions work with both: - [`rules`](../yaml/README.md#rules), which is the recommended approach, and - [`only` and `except`](../yaml/README.md#onlyexcept-basic), which are candidates for deprecation. @@ -523,15 +579,15 @@ deploy: - $STAGING ``` -Each expression provided is going to be evaluated before creating a pipeline. +Each expression provided is evaluated before a pipeline is created. -If any of the conditions in `variables` evaluates to truth when using `only`, -a new job is going to be created. If any of the expressions evaluates to truth -when `except` is being used, a job is not going to be created. +If any of the conditions in `variables` evaluates to true when using `only`, +a new job is created. If any of the expressions evaluates to true +when `except` is being used, a job is not created. -This follows usual rules for [`only` / `except` policies](../yaml/README.md#onlyexcept-advanced). +This follows the usual rules for [`only` / `except` policies](../yaml/README.md#onlyexcept-advanced). -### Supported syntax +### Syntax of environment variable expressions Below you can find supported syntax reference: @@ -679,7 +735,7 @@ If a job isn't working as expected, this can make the problem difficult to investigate; in these cases, you can enable debug tracing in `.gitlab-ci.yml`. Available on GitLab Runner v1.7+, this feature enables the shell's execution log, resulting in a verbose job log listing all commands that were run, -variables that were set, etc. +variables that were set, and so on. Before enabling this, you should ensure jobs are visible to [team members only](../../user/permissions.md#project-features). You should @@ -864,3 +920,10 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ... ``` + +## Video walkthrough of a working example + +The [Managing the Complex Configuration Data Management Monster Using GitLab](https://www.youtube.com/watch?v=v4ZOJ96hAck) video is a walkthrough of the [Complex Config Data Monorepo](https://gitlab.com/guided-explorations/config-data-top-scope/config-data-subscope/config-data-monorepo) working example project. It explains how multiple levels of group CI/CD variables can be combined with environment-scoped project variables for complex configuration of application builds or deployments. + +The example can be copied to your own group or instance for testing. More details +on what other GitLab CI patterns are demonstrated are available at the project page. diff --git a/doc/ci/variables/img/ci_job_stage_output_example.png b/doc/ci/variables/img/ci_job_stage_output_example.png Binary files differindex 056238d5693..e333da57121 100644 --- a/doc/ci/variables/img/ci_job_stage_output_example.png +++ b/doc/ci/variables/img/ci_job_stage_output_example.png diff --git a/doc/ci/variables/img/inherited_group_variables_v12_5.png b/doc/ci/variables/img/inherited_group_variables_v12_5.png Binary files differindex fd41859605f..a13ba711083 100644 --- a/doc/ci/variables/img/inherited_group_variables_v12_5.png +++ b/doc/ci/variables/img/inherited_group_variables_v12_5.png diff --git a/doc/ci/variables/img/override_value_via_manual_pipeline_output.png b/doc/ci/variables/img/override_value_via_manual_pipeline_output.png Binary files differindex 02369d57fb8..8a13bb3849e 100644 --- a/doc/ci/variables/img/override_value_via_manual_pipeline_output.png +++ b/doc/ci/variables/img/override_value_via_manual_pipeline_output.png diff --git a/doc/ci/variables/img/override_variable_manual_pipeline.png b/doc/ci/variables/img/override_variable_manual_pipeline.png Binary files differindex c77c5cb7764..de768105aec 100644 --- a/doc/ci/variables/img/override_variable_manual_pipeline.png +++ b/doc/ci/variables/img/override_variable_manual_pipeline.png diff --git a/doc/ci/variables/img/variable_types_usage_example.png b/doc/ci/variables/img/variable_types_usage_example.png Binary files differdeleted file mode 100644 index c2ae32fd048..00000000000 --- a/doc/ci/variables/img/variable_types_usage_example.png +++ /dev/null diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index f53fd371c10..d4d3a13bb2a 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -56,7 +56,7 @@ future GitLab releases.** | `CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_SHA` | 12.3 | all | The HEAD SHA of the source branch of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | | `CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME` | 12.3 | all | The target branch name of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | | `CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA` | 12.3 | all | The HEAD SHA of the target branch of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | -| `CI_JOB_ID` | 9.0 | all | The unique id of the current job that GitLab CI/CD uses internally | +| `CI_JOB_ID` | 9.0 | all | The unique ID of the current job that GitLab CI/CD uses internally | | `CI_JOB_IMAGE` | 12.9 | 12.9 | The name of the image running the CI job | | `CI_JOB_MANUAL` | 8.12 | all | The flag to indicate that job was manually started | | `CI_JOB_NAME` | 9.0 | 0.5 | The name of the job as defined in `.gitlab-ci.yml` | @@ -64,6 +64,7 @@ future GitLab releases.** | `CI_JOB_TOKEN` | 9.0 | 1.2 | Token used for authenticating with the [GitLab Container Registry](../../user/packages/container_registry/index.md) and downloading [dependent repositories](../../user/project/new_ci_build_permissions_model.md#dependent-repositories) | | `CI_JOB_JWT` | 12.10 | all | RS256 JSON web token that can be used for authenticating with third party systems that support JWT authentication, for example [HashiCorp's Vault](../examples/authenticating-with-hashicorp-vault). | | `CI_JOB_URL` | 11.1 | 0.5 | Job details URL | +| `CI_KUBERNETES_ACTIVE` | 13.0 | all | Included with the value `true` only if the pipeline has a Kubernetes cluster available for deployments. Not included if no cluster is availble. Can be used as an alternative to [`only:kubernetes`/`except:kubernetes`](../yaml/README.md#onlykubernetesexceptkubernetes) with [`rules:if`](../yaml/README.md#rulesif) | | `CI_MERGE_REQUEST_ASSIGNEES` | 11.9 | all | Comma-separated list of username(s) of assignee(s) for the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | | `CI_MERGE_REQUEST_CHANGED_PAGE_PATHS` | 12.9 | all | Comma-separated list of paths of changed pages in a deployed [Review App](../review_apps/index.md) for a [Merge Request](../merge_request_pipelines/index.md). A [Route Map](../review_apps/index.md#route-maps) must be configured. | | `CI_MERGE_REQUEST_CHANGED_PAGE_URLS` | 12.9 | all | Comma-separated list of URLs of changed pages in a deployed [Review App](../review_apps/index.md) for a [Merge Request](../merge_request_pipelines/index.md). A [Route Map](../review_apps/index.md#route-maps) must be configured. | @@ -88,13 +89,13 @@ future GitLab releases.** | `CI_NODE_TOTAL` | 11.5 | all | Total number of instances of this job running in parallel. If the job is not parallelized, this variable is set to `1`. | | `CI_PAGES_DOMAIN` | 11.8 | all | The configured domain that hosts GitLab Pages. | | `CI_PAGES_URL` | 11.8 | all | URL to GitLab Pages-built pages. Always belongs to a subdomain of `CI_PAGES_DOMAIN`. | -| `CI_PIPELINE_ID` | 8.10 | all | The unique id of the current pipeline that GitLab CI/CD uses internally | -| `CI_PIPELINE_IID` | 11.0 | all | The unique id of the current pipeline scoped to project | +| `CI_PIPELINE_ID` | 8.10 | all | The unique ID of the current pipeline that GitLab CI/CD uses internally | +| `CI_PIPELINE_IID` | 11.0 | all | The unique ID of the current pipeline scoped to project | | `CI_PIPELINE_SOURCE` | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, `pipeline`, `parent_pipeline`, `external`, `chat`, `merge_request_event`, and `external_pull_request_event`. For pipelines created before GitLab 9.5, this will show as `unknown` | | `CI_PIPELINE_TRIGGERED` | all | all | The flag to indicate that job was [triggered](../triggers/README.md) | | `CI_PIPELINE_URL` | 11.1 | 0.5 | Pipeline details URL | | `CI_PROJECT_DIR` | all | all | The full path where the repository is cloned and where the job is run. If the GitLab Runner `builds_dir` parameter is set, this variable is set relative to the value of `builds_dir`. For more information, see [Advanced configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) for GitLab Runner. | -| `CI_PROJECT_ID` | all | all | The unique id of the current project that GitLab CI/CD uses internally | +| `CI_PROJECT_ID` | all | all | The unique ID of the current project that GitLab CI/CD uses internally | | `CI_PROJECT_NAME` | 8.10 | 0.5 | The name of the directory for the project that is currently being built. For example, if the project URL is `gitlab.example.com/group-name/project-1`, the `CI_PROJECT_NAME` would be `project-1`. | | `CI_PROJECT_NAMESPACE` | 8.10 | 0.5 | The project namespace (username or groupname) that is currently being built | | `CI_PROJECT_PATH` | 8.10 | 0.5 | The namespace with project name | @@ -110,7 +111,7 @@ future GitLab releases.** | `CI_REPOSITORY_URL` | 9.0 | all | The URL to clone the Git repository | | `CI_RUNNER_DESCRIPTION` | 8.10 | 0.5 | The description of the runner as saved in GitLab | | `CI_RUNNER_EXECUTABLE_ARCH` | all | 10.6 | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) | -| `CI_RUNNER_ID` | 8.10 | 0.5 | The unique id of runner being used | +| `CI_RUNNER_ID` | 8.10 | 0.5 | The unique ID of runner being used | | `CI_RUNNER_REVISION` | all | 10.6 | GitLab Runner revision that is executing the current job | | `CI_RUNNER_SHORT_TOKEN` | all | 12.3 | First eight characters of GitLab Runner's token used to authenticate new job requests. Used as Runner's unique ID | | `CI_RUNNER_TAGS` | 8.10 | 0.5 | The defined runner tags | @@ -131,7 +132,7 @@ future GitLab releases.** | `GITLAB_CI` | all | all | Mark that job is executed in GitLab CI/CD environment | | `GITLAB_FEATURES` | 10.6 | all | The comma separated list of licensed features available for your instance and plan | | `GITLAB_USER_EMAIL` | 8.12 | all | The email of the user who started the job | -| `GITLAB_USER_ID` | 8.12 | all | The id of the user who started the job | +| `GITLAB_USER_ID` | 8.12 | all | The ID of the user who started the job | | `GITLAB_USER_LOGIN` | 10.0 | all | The login username of the user who started the job | | `GITLAB_USER_NAME` | 10.0 | all | The real name of the user who started the job | | `RESTORE_CACHE_ATTEMPTS` | 8.15 | 1.9 | Number of attempts to restore the cache running a job | diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 31459735101..c40580cbfb7 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -65,7 +65,7 @@ project namespace. For example, `https://gitlab.example.com/gitlab-org/project-1 ### Unavailable names for jobs Each job must have a unique name, but there are a few **reserved `keywords` that -cannot be used as job names**: +can't be used as job names**: - `image` - `services` @@ -90,41 +90,44 @@ A job is defined as a list of parameters that define the job's behavior. The following table lists available parameters for jobs: -| Keyword | Description | -|:---------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`script`](#script) | Shell script which is executed by Runner. | -| [`image`](#image) | Use docker images. Also available: `image:name` and `image:entrypoint`. | -| [`services`](#services) | Use docker services images. Also available: `services:name`, `services:alias`, `services:entrypoint`, and `services:command`. | -| [`before_script`](#before_script-and-after_script) | Override a set of commands that are executed before job. | -| [`after_script`](#before_script-and-after_script) | Override a set of commands that are executed after job. | -| [`stages`](#stages) | Define stages in a pipeline. | -| [`stage`](#stage) | Defines a job stage (default: `test`). | -| [`only`](#onlyexcept-basic) | Limit when jobs are created. Also available: [`only:refs`, `only:kubernetes`, `only:variables`, and `only:changes`](#onlyexcept-advanced). | -| [`except`](#onlyexcept-basic) | Limit when jobs are not created. Also available: [`except:refs`, `except:kubernetes`, `except:variables`, and `except:changes`](#onlyexcept-advanced). | -| [`rules`](#rules) | List of conditions to evaluate and determine selected attributes of a job, and whether or not it is created. May not be used alongside `only`/`except`. | -| [`tags`](#tags) | List of tags which are used to select Runner. | -| [`allow_failure`](#allow_failure) | Allow job to fail. Failed job doesn't contribute to commit status. | -| [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | -| [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. | -| [`cache`](#cache) | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, and `cache:policy`. | -| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:junit`, and `artifacts:reports:cobertura`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:codequality`, `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_management`, `artifacts:reports:performance` and `artifacts:reports:metrics`. | -| [`dependencies`](#dependencies) | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. | -| [`coverage`](#coverage) | Code coverage settings for a given job. | -| [`retry`](#retry) | When and how many times a job can be auto-retried in case of a failure. | -| [`timeout`](#timeout) | Define a custom job-level timeout that takes precedence over the project-wide setting. | -| [`parallel`](#parallel) | How many instances of a job should be run in parallel. | -| [`trigger`](#trigger) | Defines a downstream pipeline trigger. | -| [`include`](#include) | Allows this job to include external YAML files. Also available: `include:local`, `include:file`, `include:template`, and `include:remote`. | -| [`extends`](#extends) | Configuration entries that this job is going to inherit from. | -| [`pages`](#pages) | Upload the result of a job to use with GitLab Pages. | -| [`variables`](#variables) | Define job variables on a job level. | -| [`interruptible`](#interruptible) | Defines if a job can be canceled when made redundant by a newer run. | -| [`resource_group`](#resource_group) | Limit job concurrency. | +| Keyword | Description | +|:---------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`script`](#script) | Shell script which is executed by Runner. | +| [`image`](#image) | Use docker images. Also available: `image:name` and `image:entrypoint`. | +| [`services`](#services) | Use docker services images. Also available: `services:name`, `services:alias`, `services:entrypoint`, and `services:command`. | +| [`before_script`](#before_script-and-after_script) | Override a set of commands that are executed before job. | +| [`after_script`](#before_script-and-after_script) | Override a set of commands that are executed after job. | +| [`stage`](#stage) | Defines a job stage (default: `test`). | +| [`only`](#onlyexcept-basic) | Limit when jobs are created. Also available: [`only:refs`, `only:kubernetes`, `only:variables`, and `only:changes`](#onlyexcept-advanced). | +| [`except`](#onlyexcept-basic) | Limit when jobs are not created. Also available: [`except:refs`, `except:kubernetes`, `except:variables`, and `except:changes`](#onlyexcept-advanced). | +| [`rules`](#rules) | List of conditions to evaluate and determine selected attributes of a job, and whether or not it's created. May not be used alongside `only`/`except`. | +| [`tags`](#tags) | List of tags which are used to select Runner. | +| [`allow_failure`](#allow_failure) | Allow job to fail. Failed job does not contribute to commit status. | +| [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | +| [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. | +| [`cache`](#cache) | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, and `cache:policy`. | +| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:codequality`, `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_scanning`, `artifacts:reports:license_management` (removed in GitLab 13.0),`artifacts:reports:performance` and `artifacts:reports:metrics`. | +| [`dependencies`](#dependencies) | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. | +| [`coverage`](#coverage) | Code coverage settings for a given job. | +| [`retry`](#retry) | When and how many times a job can be auto-retried in case of a failure. | +| [`timeout`](#timeout) | Define a custom job-level timeout that takes precedence over the project-wide setting. | +| [`parallel`](#parallel) | How many instances of a job should be run in parallel. | +| [`trigger`](#trigger) | Defines a downstream pipeline trigger. | +| [`include`](#include) | Allows this job to include external YAML files. Also available: `include:local`, `include:file`, `include:template`, and `include:remote`. | +| [`extends`](#extends) | Configuration entries that this job is going to inherit from. | +| [`pages`](#pages) | Upload the result of a job to use with GitLab Pages. | +| [`variables`](#variables) | Define job variables on a job level. | +| [`interruptible`](#interruptible) | Defines if a job can be canceled when made redundant by a newer run. | +| [`resource_group`](#resource_group) | Limit job concurrency. | NOTE: **Note:** Parameters `types` and `type` are [deprecated](#deprecated-parameters). -## Setting default parameters +## Global parameters + +Some parameters must be defined at a global level, affecting all jobs in the pipeline. + +### Global defaults Some parameters can be set globally as the default for all jobs using the `default:` keyword. Default parameters can then be overridden by job-specific @@ -158,7 +161,7 @@ rspec 2.6: script: bundle exec rspec ``` -### `inherit` +#### `inherit` > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/207484) in GitLab 12.9. @@ -198,13 +201,13 @@ In the example below: - **will** inherit: Nothing. - `rspec`: - **will** inherit: the default `image` and the `WEBHOOK_URL` variable. - - **will not** inherit: the default `before_script` and the `DOMAIN` variable. + - will **not** inherit: the default `before_script` and the `DOMAIN` variable. - `capybara`: - **will** inherit: the default `before_script` and `image`. - - **will not** inherit: the `DOMAIN` and `WEBHOOK_URL` variables. + - will **not** inherit: the `DOMAIN` and `WEBHOOK_URL` variables. - `karma`: - **will** inherit: the default `image` and `before_script`, and the `DOMAIN` variable. - - **will not** inherit: `WEBHOOK_URL` variable. + - will **not** inherit: `WEBHOOK_URL` variable. ```yaml default: @@ -240,49 +243,274 @@ karma: script: karma ``` -## Parameter details +### `stages` -The following are detailed explanations for parameters used to configure CI/CD pipelines. +`stages` is used to define stages that can be used by jobs and is defined +globally. -### `script` +The specification of `stages` allows for having flexible multi stage pipelines. +The ordering of elements in `stages` defines the ordering of jobs' execution: -`script` is the only required keyword that a job needs. It's a shell script -which is executed by the Runner. For example: +1. Jobs of the same stage are run in parallel. +1. Jobs of the next stage are run after the jobs from the previous stage + complete successfully. + +Let's consider the following example, which defines 3 stages: ```yaml -job: - script: "bundle exec rspec" +stages: + - build + - test + - deploy ``` -[YAML anchors for scripts](#yaml-anchors-for-script) are available. +1. First, all jobs of `build` are executed in parallel. +1. If all jobs of `build` succeed, the `test` jobs are executed in parallel. +1. If all jobs of `test` succeed, the `deploy` jobs are executed in parallel. +1. If all jobs of `deploy` succeed, the commit is marked as `passed`. +1. If any of the previous jobs fails, the commit is marked as `failed` and no + jobs of further stage are executed. -This parameter can also contain several commands using an array: +There are also two edge cases worth mentioning: + +1. If no `stages` are defined in `.gitlab-ci.yml`, then the `build`, + `test` and `deploy` are allowed to be used as job's stage by default. +1. If a job does not specify a `stage`, the job is assigned the `test` stage. + +### `workflow:rules` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29654) in GitLab 12.5 + +The top-level `workflow:` key applies to the entirety of a pipeline, and will +determine whether or not a pipeline is created. It currently accepts a single +`rules:` key that operates similarly to [`rules:` defined within jobs](#rules), +enabling dynamic configuration of the pipeline. + +#### `workflow:rules` templates + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217732) in GitLab 13.0. + +We provide pre-made templates for use with your pipelines that set up `workflow: rules` +for common scenarios. Usage of these will make things easier and prevent duplicate pipelines from running. + +The [`Branch-Pipelines` template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml) +makes your pipelines run for branches and tags. + +Branch pipeline status will be displayed within merge requests that use that branch +as a source, but this pipeline type does not support any features offered by +[Merge Request Pipelines](../merge_request_pipelines/) like +[Pipelines for Merge Results](../merge_request_pipelines/#pipelines-for-merged-results-premium) +or [Merge Trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/). +Use this template if you are intentionally avoiding those features. + +It is [included](#include) as follows: ```yaml -job: - script: - - uname -a - - bundle exec rspec +include: + - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' +``` + +The [`MergeRequest-Pipelines` include](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml) sets your pipelines to run for the default branch (usually `master`), tags, and +The [`MergeRequest-Pipelines` template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml) +makes your pipelines run for the default branch (usually `master`), tags, and +all types of merge request pipelines. Use this template if you use any of the +the [Pipelines for Merge Requests features](../merge_request_pipelines/), as mentioned +above. + +It is [included](#include) as follows: + +```yaml +include: + - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' +``` + +If you prefer to define your own rules, the configuration options currently available are: + +- [`if`](#rulesif): Define a rule. +- [`when`](#when): May be set to `always` or `never` only. If not provided, the default value is `always`. + +The list of `if` rules is evaluated until a single one is matched. If none +match, the last `when` will be used: + +```yaml +workflow: + rules: + - if: $CI_COMMIT_REF_NAME =~ /-wip$/ + when: never + - if: $CI_COMMIT_TAG + when: never + - when: always ``` +### `include` + +> - Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. +> - Available for Starter, Premium and Ultimate since 10.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/42861) to GitLab Core in 11.4. + +Using the `include` keyword allows the inclusion of external YAML files. This helps +to break down the CI/CD configuration into multiple files and increases readability for long configuration files. +It's also possible to have template files stored in a central repository and projects include their +configuration files. This helps avoid duplicated configuration, for example, global default variables for all projects. + +`include` requires the external YAML file to have the extensions `.yml` or `.yaml`, +otherwise the external file won't be included. + +`include` supports the following inclusion methods: + +| Method | Description | +|:--------------------------------|:------------------------------------------------------------------| +| [`local`](#includelocal) | Include a file from the local project repository. | +| [`file`](#includefile) | Include a file from a different project repository. | +| [`remote`](#includeremote) | Include a file from a remote URL. Must be publicly accessible. | +| [`template`](#includetemplate) | Include templates which are provided by GitLab. | + NOTE: **Note:** -Sometimes, `script` commands will need to be wrapped in single or double quotes. -For example, commands that contain a colon (`:`) need to be wrapped in quotes so -that the YAML parser knows to interpret the whole thing as a string rather than -a "key: value" pair. Be careful when using special characters: -`:`, `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``. +`.gitlab-ci.yml` configuration included by all methods is evaluated at pipeline creation. +The configuration is a snapshot in time and persisted in the database. Any changes to +referenced `.gitlab-ci.yml` configuration won't be reflected in GitLab until the next pipeline is created. -If any of the script commands return an exit code different from zero, the job -will fail and further commands will not be executed. This behavior can be avoided by -storing the exit code in a variable: +The files defined in `include` are: + +- Deep merged with those in `.gitlab-ci.yml`. +- Always evaluated first and merged with the content of `.gitlab-ci.yml`, + regardless of the position of the `include` keyword. + +TIP: **Tip:** +Use merging to customize and override included CI/CD configurations with local +definitions. + +NOTE: **Note:** +Using YAML aliases across different YAML files sourced by `include` is not +supported. You must only refer to aliases in the same file. Instead +of using YAML anchors, you can use the [`extends` keyword](#extends). + +#### `include:local` + +`include:local` includes a file from the same repository as `.gitlab-ci.yml`. +It's referenced using full paths relative to the root directory (`/`). + +You can only use files that are currently tracked by Git on the same branch +your configuration file is on. In other words, when using a `include:local`, make +sure that both `.gitlab-ci.yml` and the local file are on the same branch. + +All [nested includes](#nested-includes) will be executed in the scope of the same project, +so it's possible to use local, project, remote, or template includes. + +NOTE: **Note:** +Including local files through Git submodules paths is not supported. + +Example: ```yaml -job: - script: - - false || exit_code=$? - - if [ $exit_code -ne 0 ]; then echo "Previous command failed"; fi; +include: + - local: '/templates/.gitlab-ci-template.yml' +``` + +TIP: **Tip:** +Local includes can be used as a replacement for symbolic links which are not followed. + +This can be defined as a short local include: + +```yaml +include: '.gitlab-ci-production.yml' +``` + +#### `include:file` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53903) in GitLab 11.7. + +To include files from another private project under the same GitLab instance, +use `include:file`. This file is referenced using full paths relative to the +root directory (`/`). For example: + +```yaml +include: + - project: 'my-group/my-project' + file: '/templates/.gitlab-ci-template.yml' ``` +You can also specify `ref`, with the default being the `HEAD` of the project: + +```yaml +include: + - project: 'my-group/my-project' + ref: master + file: '/templates/.gitlab-ci-template.yml' + + - project: 'my-group/my-project' + ref: v1.0.0 + file: '/templates/.gitlab-ci-template.yml' + + - project: 'my-group/my-project' + ref: 787123b47f14b552955ca2786bc9542ae66fee5b # Git SHA + file: '/templates/.gitlab-ci-template.yml' +``` + +All [nested includes](#nested-includes) will be executed in the scope of the target project, +so it's possible to use local (relative to target project), project, remote +or template includes. + +#### `include:remote` + +`include:remote` can be used to include a file from a different location, +using HTTP/HTTPS, referenced by using the full URL. The remote file must be +publicly accessible through a simple GET request as authentication schemas +in the remote URL are not supported. For example: + +```yaml +include: + - remote: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-template.yml' +``` + +All [nested includes](#nested-includes) will be executed without context as public user, so only another remote +or public project, or template, is allowed. + +#### `include:template` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53445) in GitLab 11.7. + +`include:template` can be used to include `.gitlab-ci.yml` templates that are +[shipped with GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates). + +For example: + +```yaml +# File sourced from GitLab's template collection +include: + - template: Auto-DevOps.gitlab-ci.yml +``` + +Multiple `include:template` files: + +```yaml +include: + - template: Android-Fastlane.gitlab-ci.yml + - template: Auto-DevOps.gitlab-ci.yml +``` + +All [nested includes](#nested-includes) will be executed only with the permission of the user, +so it's possible to use project, remote or template includes. + +#### Nested includes + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/56836) in GitLab 11.9. + +Nested includes allow you to compose a set of includes. + +A total of 100 includes is allowed, but duplicate includes are considered a configuration error. + +Since [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/28212), the time limit +for resolving all files is 30 seconds. + +#### Additional `includes` examples + +There is a list of [additional `includes` examples](includes.md) available. + +## Parameter details + +The following are detailed explanations for parameters used to configure CI/CD pipelines. + ### `image` Used to specify [a Docker image](../docker/using_docker_images.md#what-is-an-image) to use for the job. @@ -304,7 +532,7 @@ An [extended docker configuration option](../docker/using_docker_images.md#exten For more information, see [Available settings for `image`](../docker/using_docker_images.md#available-settings-for-image). -### `services` +#### `services` Used to specify a [service Docker image](../docker/using_docker_images.md#what-is-a-service), linked to a base image specified in [`image`](#image). @@ -314,31 +542,70 @@ For: - Detailed usage information, refer to [Docker integration](../docker/README.md) documentation. - For example services, see [GitLab CI/CD Services](../services/README.md). -#### `services:name` +##### `services:name` An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). -#### `services:alias` +##### `services:alias` An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). -#### `services:entrypoint` +##### `services:entrypoint` An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). -#### `services:command` +##### `services:command` An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). -### `before_script` and `after_script` +### `script` + +`script` is the only required keyword that a job needs. It's a shell script +which is executed by the Runner. For example: + +```yaml +job: + script: "bundle exec rspec" +``` + +[YAML anchors for scripts](#yaml-anchors-for-script) are available. + +This parameter can also contain several commands using an array: + +```yaml +job: + script: + - uname -a + - bundle exec rspec +``` + +NOTE: **Note:** +Sometimes, `script` commands will need to be wrapped in single or double quotes. +For example, commands that contain a colon (`:`) need to be wrapped in quotes so +that the YAML parser knows to interpret the whole thing as a string rather than +a "key: value" pair. Be careful when using special characters: +`:`, `{`, `}`, `[`, `]`, `,`, `&`, `*`, `#`, `?`, `|`, `-`, `<`, `>`, `=`, `!`, `%`, `@`, `` ` ``. + +If any of the script commands return an exit code different from zero, the job +will fail and further commands won't be executed. This behavior can be avoided by +storing the exit code in a variable: + +```yaml +job: + script: + - false || exit_code=$? + - if [ $exit_code -ne 0 ]; then echo "Previous command failed"; fi; +``` + +#### `before_script` and `after_script` > Introduced in GitLab 8.7 and requires GitLab Runner v1.2. @@ -362,7 +629,7 @@ Scripts specified in `after_script` are executed in a new shell, separate from a software installed by a `before_script` or `script` script. - Have a separate timeout, which is hard coded to 5 minutes. See [related issue](https://gitlab.com/gitlab-org/gitlab-runner/issues/2716) for details. -- Do not affect the job's exit code. If the `script` section succeeds and the +- Don't affect the job's exit code. If the `script` section succeeds and the `after_script` times out or fails, the job will exit with code `0` (`Job Succeeded`). It's possible to overwrite a globally defined `before_script` or `after_script` @@ -384,39 +651,53 @@ job: [YAML anchors for `before_script` and `after_script`](#yaml-anchors-for-before_script-and-after_script) are available. -### `stages` - -`stages` is used to define stages that can be used by jobs and is defined -globally. - -The specification of `stages` allows for having flexible multi stage pipelines. -The ordering of elements in `stages` defines the ordering of jobs' execution: - -1. Jobs of the same stage are run in parallel. -1. Jobs of the next stage are run after the jobs from the previous stage - complete successfully. +### `stage` -Let's consider the following example, which defines 3 stages: +`stage` is defined per-job and relies on [`stages`](#stages) which is defined +globally. It allows to group jobs into different stages, and jobs of the same +`stage` are executed in parallel (subject to [certain conditions](#using-your-own-runners)). For example: ```yaml stages: - build - test - deploy + +job 0: + stage: .pre + script: make something useful before build stage + +job 1: + stage: build + script: make build dependencies + +job 2: + stage: build + script: make build artifacts + +job 3: + stage: test + script: make test + +job 4: + stage: deploy + script: make deploy + +job 5: + stage: .post + script: make something useful at the end of pipeline ``` -1. First, all jobs of `build` are executed in parallel. -1. If all jobs of `build` succeed, the `test` jobs are executed in parallel. -1. If all jobs of `test` succeed, the `deploy` jobs are executed in parallel. -1. If all jobs of `deploy` succeed, the commit is marked as `passed`. -1. If any of the previous jobs fails, the commit is marked as `failed` and no - jobs of further stage are executed. +#### Using your own Runners -There are also two edge cases worth mentioning: +When using your own Runners, GitLab Runner runs only one job at a time by default (see the +`concurrent` flag in [Runner global settings](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section) +for more information). -1. If no `stages` are defined in `.gitlab-ci.yml`, then the `build`, - `test` and `deploy` are allowed to be used as job's stage by default. -1. If a job doesn't specify a `stage`, the job is assigned the `test` stage. +Jobs will run on your own Runners in parallel only if: + +- Run on different Runners. +- The Runner's `concurrent` setting has been changed. #### `.pre` and `.post` @@ -429,12 +710,12 @@ The following stages are available to every pipeline: User-defined stages are executed after `.pre` and before `.post`. -The order of `.pre` and `.post` cannot be changed, even if defined out of order in `.gitlab-ci.yml`. +The order of `.pre` and `.post` can't be changed, even if defined out of order in `.gitlab-ci.yml`. For example, the following are equivalent configuration: - Configured in order: - ```yml + ```yaml stages: - .pre - a @@ -444,7 +725,7 @@ For example, the following are equivalent configuration: - Configured out of order: - ```yml + ```yaml stages: - a - .pre @@ -454,69 +735,390 @@ For example, the following are equivalent configuration: - Not explicitly configured: - ```yml + ```yaml stages: - a - b ``` NOTE: **Note:** -A pipeline will not be created if it only contains jobs in `.pre` or `.post` stages. +A pipeline won't be created if it only contains jobs in `.pre` or `.post` stages. -### `stage` +### `extends` -`stage` is defined per-job and relies on [`stages`](#stages) which is defined -globally. It allows to group jobs into different stages, and jobs of the same -`stage` are executed in parallel (subject to [certain conditions](#using-your-own-runners)). For example: +> Introduced in GitLab 11.3. + +`extends` defines entry names that a job that uses `extends` is going to +inherit from. + +It's an alternative to using [YAML anchors](#anchors) and is a little +more flexible and readable: ```yaml -stages: - - build - - test - - deploy +.tests: + script: rake test + stage: test + only: + refs: + - branches -job 0: - stage: .pre - script: make something useful before build stage +rspec: + extends: .tests + script: rake rspec + only: + variables: + - $RSPEC +``` -job 1: - stage: build - script: make build dependencies +In the example above, the `rspec` job inherits from the `.tests` template job. +GitLab will perform a reverse deep merge based on the keys. GitLab will: -job 2: - stage: build - script: make build artifacts +- Merge the `rspec` contents into `.tests` recursively. +- Not merge the values of the keys. -job 3: +This results in the following `rspec` job: + +```yaml +rspec: + script: rake rspec stage: test - script: make test + only: + refs: + - branches + variables: + - $RSPEC +``` -job 4: - stage: deploy - script: make deploy +NOTE: **Note:** +Note that `script: rake test` has been overwritten by `script: rake rspec`. -job 5: - stage: .post - script: make something useful at the end of pipeline +If you do want to include the `rake test`, see [`before_script` and `after_script`](#before_script-and-after_script). + +`.tests` in this example is a [hidden job](#hide-jobs), but it's +possible to inherit from regular jobs as well. + +`extends` supports multi-level inheritance, however it's not recommended to +use more than three levels. The maximum nesting level that is supported is 10. +The following example has two levels of inheritance: + +```yaml +.tests: + only: + - pushes + +.rspec: + extends: .tests + script: rake rspec + +rspec 1: + variables: + RSPEC_SUITE: '1' + extends: .rspec + +rspec 2: + variables: + RSPEC_SUITE: '2' + extends: .rspec + +spinach: + extends: .tests + script: rake spinach ``` -#### Using your own Runners +In GitLab 12.0 and later, it's also possible to use multiple parents for +`extends`. The algorithm used for merge is "closest scope wins", so +keys from the last member will always shadow anything defined on other +levels. For example: -When using your own Runners, GitLab Runner runs only one job at a time by default (see the -`concurrent` flag in [Runner global settings](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section) -for more information). +```yaml +.only-important: + only: + - master + - stable + tags: + - production -Jobs will run on your own Runners in parallel only if: +.in-docker: + tags: + - docker + image: alpine -- Run on different Runners. -- The Runner's `concurrent` setting has been changed. +rspec: + extends: + - .only-important + - .in-docker + script: + - rake rspec +``` + +This results in the following `rspec` job: + +```yaml +rspec: + only: + - master + - stable + tags: + - docker + image: alpine + script: + - rake rspec +``` + +#### Using `extends` and `include` together + +`extends` works across configuration files combined with `include`. + +For example, if you have a local `included.yml` file: + +```yaml +.template: + script: + - echo Hello! +``` + +Then, in `.gitlab-ci.yml` you can use it like this: + +```yaml +include: included.yml + +useTemplate: + image: alpine + extends: .template +``` + +This will run a job called `useTemplate` that runs `echo Hello!` as defined in +the `.template` job, and uses the `alpine` Docker image as defined in the local job. + +### `rules` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27863) in GitLab 12.3. + +`rules` allows for a list of individual rule objects to be evaluated +*in order*, until one matches and dynamically provides attributes to the job. + +CAUTION: **Caution:** +`rules` can't be used in combination with `only/except` as it is a replacement for that functionality. If you attempt to do this, the linter will return a +`key may not be used with rules` error. + +#### Key details when using `rules` + +A very important difference between `rules` and `only/except`, is that jobs defined +with `rules` trigger merge request pipelines by default, but `only/except` jobs do not. +This may be surprising if migrating from `only` and `except`, so new users of `rules` +can use one of the [`workflow: rules` templates](#workflowrules-templates) to get started. +This will ensure that the behavior is more stable as you start adding additional `rules` +blocks, and will avoid issues like creating a duplicate, merge request (detached) pipeline. + +We don't recomment mixing `only/except` jobs with `rules` jobs in the same pipeline. +It may not cause YAML errors, but debugging the exact execution behavior can be complex +due to the different default behaviors of `only/except` and `rules`. + +### Rules clauses + +Available rule clauses include: + +- [`if`](#rulesif) (similar to [`only:variables`](#onlyvariablesexceptvariables)) +- [`changes`](#ruleschanges) (same as [`only:changes`](#onlychangesexceptchanges)) +- [`exists`](#rulesexists) + +For example, using `if`. This configuration specifies that `job` should be built +and run for every pipeline on merge requests targeting `master`, regardless of +the status of other builds: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' + when: always + - if: '$VAR =~ /pattern/' + when: manual + - when: on_success +``` + +In this example, if the first rule: + +- Matches, the job will be given the `when:always` attribute. +- Does not match, the second and third rules will be evaluated sequentially + until a match is found. That is, the job will be given either the: + - `when: manual` attribute if the second rule matches. **The stage won't complete until this manual job is triggered and completes successfully.** + - `when: on_success` attribute if the second rule does not match. The third + rule will always match when reached because it has no conditional clauses. + +#### `rules:if` + +`rules:if` differs slightly from `only:variables` by accepting only a single +expression string, rather than an array of them. Any set of expressions to be +evaluated should be conjoined into a single expression using `&&` or `||`, and use +the [variable matching syntax](../variables/README.md#syntax-of-environment-variable-expressions). + +For example: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' # This rule will be evaluated + when: always + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/' # This rule will only be evaluated if the target branch is not "master" + when: manual + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' # If neither of the first two match but the simple presence does, we set to "on_success" by default +``` + +If none of the provided rules match, the job will be set to `when:never`, and +not included in the pipeline. If `rules:when` is not included in the configuration +at all, the behavior defaults to `job:when`, which continues to default to +`on_success`. + +#### `rules:changes` + +`rules: changes` works exactly the same way as `only: changes` and `except: changes`, +accepting an array of paths. Similarly, it will always return true if there is no +Git push event. See [`only/except: changes`](#onlychangesexceptchanges) for more information. + +For example: + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. + - Dockerfile + when: manual + - if: '$VAR == "string value"' + when: manual # Will include the job and set to when:manual if the expression evaluates to true, after the `changes:` rule fails to match. + - when: on_success # If neither of the first rules match, set to on_success +``` + +In this example, a job either set to: + +- Run manually if `Dockerfile` has changed OR `$VAR == "string value"`. +- `when:on_success` by the last rule, where no earlier clauses evaluate to true. + +#### `rules:exists` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. + +`exists` accepts an array of paths and will match if any of these paths exist +as files in the repository. + +For example: + +```yaml +job: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - exists: + - Dockerfile +``` + +You can also use glob patterns to match multiple files in any directory within +the repository. + +For example: + +```yaml +job: + script: bundle exec rspec + rules: + - exists: + - spec/**.rb +``` + +NOTE: **Note:** +For performance reasons, using `exists` with patterns is limited to 10000 +checks. After the 10000th check, rules with patterned globs will always match. + +#### `rules:allow_failure` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30235) in GitLab 12.8. + +You can use [`allow_failure: true`](#allow_failure) within `rules:` to allow a job to fail, or a manual job to +wait for action, without stopping the pipeline itself. All jobs using `rules:` default to `allow_failure: false` +if `allow_failure:` is not defined. + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' + when: manual + allow_failure: true +``` + +In this example, if the first rule matches, then the job will have `when: manual` and `allow_failure: true`. + +#### Complex rule clauses + +To conjoin `if`, `changes`, and `exists` clauses with an AND, use them in the +same rule. + +In the following example: + +- We run the job manually if `Dockerfile` or any file in `docker/scripts/` + has changed AND `$VAR == "string value"`. +- Otherwise, the job won't be included in the pipeline. + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: '$VAR == "string value"' + changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. + - Dockerfile + - docker/scripts/* + when: manual + # - when: never would be redundant here, this is implied any time rules are listed. +``` + +The only clauses currently available are: + +- `if` +- `changes` +- `exists` + +Keywords such as `branches` or `refs` that are currently available for +`only`/`except` are not yet available in `rules` as they are being individually +considered for their usage and behavior in this context. Future keyword improvements +are being discussed in our [epic for improving `rules`](https://gitlab.com/groups/gitlab-org/-/epics/2783), +where anyone can add suggestions or requests. + +#### Permitted attributes + +The only job attributes currently set by `rules` are: + +- `when`. +- `start_in`, if `when` is set to `delayed`. +- `allow_failure`. + +A job will be included in a pipeline if `when` is evaluated to any value +except `never`. + +Delayed jobs require a `start_in` value, so rule objects do as well. For +example: + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - changes: # Will include the job and delay 3 hours when the Dockerfile has changed + - Dockerfile + when: delayed + start_in: '3 hours' + - when: on_success # Otherwise include the job and set to run normally +``` + +Additional job configuration may be added to rules in the future. If something +useful is not available, please +[open an issue](https://gitlab.com/gitlab-org/gitlab/issues). ### `only`/`except` (basic) NOTE: **Note:** -The [`rules`](#rules) syntax is now the preferred method of setting job policies. -`only` and `except` are [candidates for deprecation](https://gitlab.com/gitlab-org/gitlab/issues/27449), -and may be removed in the future. +The [`rules`](#rules) syntax is an improved, more powerful solution for defining +when jobs should run or not. Consider using `rules` instead of `only/except` to get +the most out of your pipelines. `only` and `except` are two parameters that set a job policy to limit when jobs are created: @@ -604,7 +1206,7 @@ The above example will run `job` for all branches on `gitlab-org/gitlab`, except `master` and those with names prefixed with `release/`. If a job does not have an `only` rule, `only: ['branches', 'tags']` is set by -default. If it doesn't have an `except` rule, it is empty. +default. If it does not have an `except` rule, it's empty. For example, @@ -643,7 +1245,7 @@ matching only a substring of the tag name or branch name. For example, `/^issue-.*$/` is equivalent to `/^issue-/`, while just `/issue/` would also match a branch called `severe-issues`. -### Supported `only`/`except` regexp syntax +#### Supported `only`/`except` regexp syntax CAUTION: **Warning:** This is a breaking change that was introduced with GitLab 11.9.4. @@ -667,7 +1269,7 @@ Feature.enable(:allow_unsafe_ruby_regexp) ### `only`/`except` (advanced) CAUTION: **Warning:** -This is an _alpha_ feature, and it is subject to change at any time without +This is an _alpha_ feature, and is subject to change at any time without prior notice! GitLab supports both simple and complex strategies, so it's possible to use an @@ -719,7 +1321,7 @@ This means the keys are treated as if joined by an OR. This relationship could b In the example below, the `test` job will **not** be created when **any** of the following are true: - The pipeline runs for the `master`. -- There are changes to the `README.md` file in the root directory of the repo. +- There are changes to the `README.md` file in the root directory of the repository. ```yaml test: @@ -813,7 +1415,7 @@ This means the `only:changes` policy is useful for pipelines where: If there is no Git push event, such as for pipelines with [sources other than the three above](../variables/predefined_variables.md#variables-reference), -`changes` cannot determine if a given file is new or old, and will always +`changes` can't determine if a given file is new or old, and will always return true. A basic example of using `only: changes`: @@ -840,10 +1442,10 @@ commits contains changes to any of the following: CAUTION: **Warning:** If using `only:changes` with [only allow merge requests to be merged if the pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds), -undesired behavior could result if you do not [also use `only:merge_requests`](#using-onlychanges-with-pipelines-for-merge-requests). +undesired behavior could result if you don't [also use `only:merge_requests`](#using-onlychanges-with-pipelines-for-merge-requests). You can also use glob patterns to match multiple files in either the root directory -of the repo, or in _any_ directory within the repo, but they must be wrapped +of the repository, or in _any_ directory within the repository, but they must be wrapped in double quotes or GitLab will fail to parse the `.gitlab-ci.yml`. For example: ```yaml @@ -856,7 +1458,7 @@ test: ``` The following example will skip the `build` job if a change is detected in any file -in the root directory of the repo with a `.md` extension: +in the root directory of the repository with a `.md` extension: ```yaml build: @@ -877,7 +1479,7 @@ There are some points to be aware of when ##### Using `only:changes` with pipelines for merge requests With [pipelines for merge requests](../merge_request_pipelines/index.md), -it is possible to define a job to be created based on files modified +it's possible to define a job to be created based on files modified in a merge request. In order to deduce the correct base SHA of the source branch, we recommend combining @@ -919,7 +1521,7 @@ docker build service one: In the example above, a pipeline could fail due to changes to a file in `service-one/**/*`. A later commit could then be pushed that does not include any changes to this file, -but includes changes to the `Dockerfile`, and this pipeline could pass because it is only +but includes changes to the `Dockerfile`, and this pipeline could pass because it's only testing the changes to the `Dockerfile`. GitLab checks the **most recent pipeline**, that **passed**, and will show the merge request as mergeable, despite the earlier failed pipeline caused by a change that was not yet corrected. @@ -944,279 +1546,189 @@ This could result in some unexpected behavior, including: All files are considered to have "changed" when a scheduled pipeline runs. -### `rules` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29011) in GitLab 12.3. +### `needs` -`rules` allows for a list of individual rule objects to be evaluated -*in order*, until one matches and dynamically provides attributes to the job. -Note that `rules` cannot be used in combination with `only/except` since it is intended -to replace that functionality. If you attempt to do this the linter will return a -`key may not be used with rules` error. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47063) in GitLab 12.2. +> - In GitLab 12.3, maximum number of jobs in `needs` array raised from five to 50. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30631) in GitLab 12.8, `needs: []` lets jobs start immediately. -Available rule clauses include: +The `needs:` keyword enables executing jobs out-of-order, allowing you to implement +a [directed acyclic graph](../directed_acyclic_graph/index.md) in your `.gitlab-ci.yml`. -- [`if`](#rulesif) (similar to [`only:variables`](#onlyvariablesexceptvariables)) -- [`changes`](#ruleschanges) (same as [`only:changes`](#onlychangesexceptchanges)) -- [`exists`](#rulesexists) +This lets you run some jobs without waiting for other ones, disregarding stage ordering +so you can have multiple stages running concurrently. -For example, using `if`. This configuration specifies that `job` should be built -and run for every pipeline on merge requests targeting `master`, regardless of -the status of other builds: +Let's consider the following example: ```yaml -job: - script: "echo Hello, Rules!" - rules: - - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' - when: always - - if: '$VAR =~ /pattern/' - when: manual - - when: on_success -``` - -In this example, if the first rule: - -- Matches, the job will be given the `when:always` attribute. -- Does not match, the second and third rules will be evaluated sequentially - until a match is found. That is, the job will be given either the: - - `when: manual` attribute if the second rule matches. **The stage will not complete until this manual job is triggered and completes successfully.** - - `when: on_success` attribute if the second rule does not match. The third - rule will always match when reached because it has no conditional clauses. - -#### `rules:if` - -`rules:if` differs slightly from `only:variables` by accepting only a single -expression string, rather than an array of them. Any set of expressions to be -evaluated should be conjoined into a single expression using `&&` or `||`, and use -the [variable matching syntax](../variables/README.md#supported-syntax). +linux:build: + stage: build -For example: +mac:build: + stage: build -```yaml -job: - script: "echo Hello, Rules!" - rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' # This rule will be evaluated - when: always - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/' # This rule will only be evaluated if the target branch is not "master" - when: manual - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' # If neither of the first two match but the simple presence does, we set to "on_success" by default -``` +lint: + stage: test + needs: [] -If none of the provided rules match, the job will be set to `when:never`, and -not included in the pipeline. If `rules:when` is not included in the configuration -at all, the behavior defaults to `job:when`, which continues to default to -`on_success`. +linux:rspec: + stage: test + needs: ["linux:build"] -#### `rules:changes` +linux:rubocop: + stage: test + needs: ["linux:build"] -`rules: changes` works exactly the same way as `only: changes` and `except: changes`, -accepting an array of paths. Similarly, it will always return true if there is no -Git push event. See [`only/except: changes`](#onlychangesexceptchanges) for more information. +mac:rspec: + stage: test + needs: ["mac:build"] -For example: +mac:rubocop: + stage: test + needs: ["mac:build"] -```yaml -docker build: - script: docker build -t my-image:$CI_COMMIT_REF_SLUG . - rules: - - changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. - - Dockerfile - when: manual - - if: '$VAR == "string value"' - when: manual # Will include the job and set to when:manual if the expression evaluates to true, after the `changes:` rule fails to match. - - when: on_success # If neither of the first rules match, set to on_success +production: + stage: deploy ``` -In this example, a job either set to: +This example creates four paths of execution: -- Run manually if `Dockerfile` has changed OR `$VAR == "string value"`. -- `when:on_success` by the last rule, where no earlier clauses evaluate to true. +- Linter: the `lint` job will run immediately without waiting for the `build` stage to complete because it has no needs (`needs: []`). -#### `rules:exists` +- Linux path: the `linux:rspec` and `linux:rubocop` jobs will be run as soon + as the `linux:build` job finishes without waiting for `mac:build` to finish. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16574) in GitLab 12.4. +- macOS path: the `mac:rspec` and `mac:rubocop` jobs will be run as soon + as the `mac:build` job finishes, without waiting for `linux:build` to finish. -`exists` accepts an array of paths and will match if any of these paths exist -as files in the repository. +- The `production` job will be executed as soon as all previous jobs + finish; in this case: `linux:build`, `linux:rspec`, `linux:rubocop`, + `mac:build`, `mac:rspec`, `mac:rubocop`. -For example: +#### Requirements and limitations -```yaml -job: - script: docker build -t my-image:$CI_COMMIT_REF_SLUG . - rules: - - exists: - - Dockerfile -``` +- If `needs:` is set to point to a job that is not instantiated + because of `only/except` rules or otherwise does not exist, the + pipeline will be created with YAML error. +- The maximum number of jobs that a single job can need in the `needs:` array is limited: + - For GitLab.com, the limit is ten. For more information, see our + [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/7541). + - For self-managed instances, the limit is: + - 10, if the `ci_dag_limit_needs` feature flag is enabled (default). + - 50, if the `ci_dag_limit_needs` feature flag is disabled. +- If `needs:` refers to a job that is marked as `parallel:`. + the current job will depend on all parallel jobs created. +- `needs:` is similar to `dependencies:` in that it needs to use jobs from prior stages, + meaning it's impossible to create circular dependencies. Depending on jobs in the + current stage is not possible either, but support [is planned](https://gitlab.com/gitlab-org/gitlab/issues/30632). +- Related to the above, stages must be explicitly defined for all jobs + that have the keyword `needs:` or are referred to by one. -You can also use glob patterns to match multiple files in any directory within -the repository. +##### Changing the `needs:` job limit -For example: +The maximum number of jobs that can be defined within `needs:` defaults to 10, but +can be changed to 50 via a feature flag. To change the limit to 50, +[start a Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) +and run: -```yaml -job: - script: bundle exec rspec - rules: - - exists: - - spec/**.rb +```ruby +Feature::disable(:ci_dag_limit_needs) ``` -NOTE: **Note:** -For performance reasons, using `exists` with patterns is limited to 10000 -checks. After the 10000th check, rules with patterned globs will always match. - -#### `rules:allow_failure` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30235) in GitLab 12.8. - -You can use [`allow_failure: true`](#allow_failure) within `rules:` to allow a job to fail, or a manual job to -wait for action, without stopping the pipeline itself. All jobs using `rules:` default to `allow_failure: false` -if `allow_failure:` is not defined. +To set it back to 10, run the opposite command: -```yaml -job: - script: "echo Hello, Rules!" - rules: - - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"' - when: manual - allow_failure: true +```ruby +Feature::enable(:ci_dag_limit_needs) ``` -In this example, if the first rule matches, then the job will have `when: manual` and `allow_failure: true`. - -#### Exclude jobs with `rules:` from certain pipelines - -Jobs with `rules:` can cause two pipelines to be created unexpectedly: - -- One pipeline from pushing a commit to a branch. -- A second ["detached" pipeline for a merge request](../merge_request_pipelines/index.md). - -`only` and `except` jobs do not trigger merge request pipelines by default, but this -is not the case for jobs with `rules:`, which may be surprising if migrating from `only` -and `except` to `rules:`. - -If you are using `rules:` and you see two pipelines for commits to branches that have -a merge request, you have two options: - -- Individually exclude each job that uses `rules:` from merge request pipelines. The - example below will cause the job to **not** run in *pipelines for merge requests*, - but it **will** run in pipelines for *new tags and pipelines running on branch refs*: +#### Artifact downloads with `needs` - ```yaml - job: - rules: - - if: $CI_MERGE_REQUEST_ID - when: never - - when: manual - script: - - echo hello - ``` +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14311) in GitLab v12.6. -- Add a global [`workflow: rules`](#workflowrules) to allow pipelines in only certain - situations. The example below will only run pipelines for merge requests, new tags and - changes to master. It will **not** run any pipelines *on any branch except master*, but - it will run **detached merge request pipelines** for any merge request, targeting any branch: +When using `needs`, artifact downloads are controlled with `artifacts: true` or `artifacts: false`. +The `dependencies` keyword should not be used with `needs`, as this is deprecated since GitLab 12.6. - ```yaml - workflow: - rules: - - if: $CI_MERGE_REQUEST_ID - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_BRANCH == "master" - ``` +In the example below, the `rspec` job will download the `build_job` artifacts, while the +`rubocop` job won't: -#### Complex rule clauses +```yaml +build_job: + stage: build + artifacts: + paths: + - binaries/ -To conjoin `if`, `changes`, and `exists` clauses with an AND, use them in the -same rule. +rspec: + stage: test + needs: + - job: build_job + artifacts: true -In the following example: +rubocop: + stage: test + needs: + - job: build_job + artifacts: false +``` -- We run the job manually if `Dockerfile` or any file in `docker/scripts/` - has changed AND `$VAR == "string value"`. -- Otherwise, the job will not be included in the pipeline. +Additionally, in the three syntax examples below, the `rspec` job will download the artifacts +from all three `build_jobs`, as `artifacts` is true for `build_job_1`, and will +**default** to true for both `build_job_2` and `build_job_3`. ```yaml -docker build: - script: docker build -t my-image:$CI_COMMIT_REF_SLUG . - rules: - - if: '$VAR == "string value"' - changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. - - Dockerfile - - docker/scripts/* - when: manual - # - when: never would be redundant here, this is implied any time rules are listed. +rspec: + needs: + - job: build_job_1 + artifacts: true + - job: build_job_2 + - build_job_3 ``` -The only clauses currently available are: - -- `if` -- `changes` -- `exists` - -Keywords such as `branches` or `refs` that are currently available for -`only`/`except` are not yet available in `rules` as they are being individually -considered for their usage and behavior in this context. - -#### Permitted attributes - -The only job attributes currently set by `rules` are: - -- `when`. -- `start_in`, if `when` is set to `delayed`. -- `allow_failure`. +#### Cross project artifact downloads with `needs` **(PREMIUM)** -A job will be included in a pipeline if `when` is evaluated to any value -except `never`. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14311) in GitLab v12.7. -Delayed jobs require a `start_in` value, so rule objects do as well. For -example: +`needs` can be used to download artifacts from up to five jobs in pipelines on +[other refs in the same project](#artifact-downloads-between-pipelines-in-the-same-project), +or pipelines in different projects: ```yaml -docker build: - script: docker build -t my-image:$CI_COMMIT_REF_SLUG . - rules: - - changes: # Will include the job and delay 3 hours when the Dockerfile has changed - - Dockerfile - when: delayed - start_in: '3 hours' - - when: on_success # Otherwise include the job and set to run normally +build_job: + stage: build + script: + - ls -lhR + needs: + - project: group/project-name + job: build-1 + ref: master + artifacts: true ``` -Additional job configuration may be added to rules in the future. If something -useful isn't available, please -[open an issue](https://gitlab.com/gitlab-org/gitlab/issues). - -### `workflow:rules` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29654) in GitLab 12.5 - -The top-level `workflow:` key applies to the entirety of a pipeline, and will -determine whether or not a pipeline is created. It currently accepts a single -`rules:` key that operates similarly to [`rules:` defined within jobs](#rules), -enabling dynamic configuration of the pipeline. - -The configuration options currently available for `workflow:rules` are: +`build_job` will download the artifacts from the latest successful `build-1` job +on the `master` branch in the `group/project-name` project. -- [`if`](#rulesif): Define a rule. -- [`when`](#when): May be set to `always` or `never` only. If not provided, the default value is `always`. +##### Artifact downloads between pipelines in the same project -The list of `if` rules is evaluated until a single one is matched. If none -match, the last `when` will be used: +`needs` can be used to download artifacts from different pipelines in the current project +by setting the `project` keyword as the current project's name, and specifying a ref. +In the example below, `build_job` will download the artifacts for the latest successful +`build-1` job with the `other-ref` ref: ```yaml -workflow: - rules: - - if: $CI_COMMIT_REF_NAME =~ /-wip$/ - when: never - - if: $CI_COMMIT_TAG - when: never - - when: always +build_job: + stage: build + script: + - ls -lhR + needs: + - project: group/same-project-name + job: build-1 + ref: other-ref + artifacts: true ``` +NOTE: **Note:** +Downloading artifacts from jobs that are run in [`parallel:`](#parallel) is not supported. + ### `tags` `tags` is used to select specific Runners from the list of all Runners that are @@ -1277,7 +1789,7 @@ show the same orange warning. However, the associated commit will be marked "passed", without warnings. In the example below, `job1` and `job2` will run in parallel, but if `job1` -fails, it will not stop the next stage from running, since it's marked with +fails, it won't stop the next stage from running, since it's marked with `allow_failure: true`: ```yaml @@ -1372,21 +1884,21 @@ Manual actions are a special type of job that are not executed automatically, they need to be explicitly started by a user. An example usage of manual actions would be a deployment to a production environment. Manual actions can be started from the pipeline, job, environment, and deployment views. Read more at the -[environments documentation](../environments.md#configuring-manual-deployments). +[environments documentation](../environments/index.md#configuring-manual-deployments). Manual actions can be either optional or blocking. Blocking manual actions will block the execution of the pipeline at the stage this action is defined in. It's possible to resume execution of the pipeline when someone executes a blocking manual action by clicking a _play_ button. -When a pipeline is blocked, it will not be merged if Merge When Pipeline Succeeds +When a pipeline is blocked, it won't be merged if Merge When Pipeline Succeeds is set. Blocked pipelines also do have a special status, called _manual_. When the `when:manual` syntax is used, manual actions are non-blocking by -default. If you want to make manual action blocking, it is necessary to add +default. If you want to make manual action blocking, it's necessary to add `allow_failure: false` to the job's definition in `.gitlab-ci.yml`. Optional manual actions have `allow_failure: true` set by default and their -Statuses do not contribute to the overall pipeline status. So, if a manual +Statuses don't contribute to the overall pipeline status. So, if a manual action fails, the pipeline will eventually succeed. NOTE: **Note:** @@ -1396,7 +1908,7 @@ Manual actions are considered to be write actions, so permissions for [protected branches](../../user/project/protected_branches.md) are used when a user wants to trigger an action. In other words, in order to trigger a manual action assigned to a branch that the pipeline is running for, the user needs to -have the ability to merge to this branch. It is possible to use protected environments +have the ability to merge to this branch. It's possible to use protected environments to more strictly [protect manual deployments](#protecting-manual-jobs-premium) from being run by unauthorized users. @@ -1409,7 +1921,7 @@ being used. It's possible to use [protected environments](../environments/protected_environments.md) to define a precise list of users authorized to run a manual job. By allowing only -users associated with a protected environment to trigger manual jobs, it is possible +users associated with a protected environment to trigger manual jobs, it's possible to implement some special use cases, such as: - More precisely limiting who can deploy to an environment. @@ -1439,13 +1951,13 @@ To do this, you must: who are always able to use protected environments. Additionally, if a manual job is defined as blocking by adding `allow_failure: false`, -the next stages of the pipeline will not run until the manual job is triggered. This +the next stages of the pipeline won't run until the manual job is triggered. This can be used as a way to have a defined list of users allowed to "approve" later pipeline stages by triggering the blocking manual job. #### `when:delayed` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21767) in GitLab 11.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51352) in GitLab 11.4. Delayed job are for executing scripts after a certain period. This is useful if you want to avoid jobs entering `pending` state immediately. @@ -1459,11 +1971,11 @@ provided. `start_in` key must be less than or equal to one week. Examples of val - `1 day` - `1 week` -When there is a delayed job in a stage, the pipeline will not progress until the delayed job has finished. +When there is a delayed job in a stage, the pipeline won't progress until the delayed job has finished. This means this keyword can also be used for inserting delays between different stages. The timer of a delayed job starts immediately after the previous stage has completed. -Similar to other types of jobs, a delayed job's timer will not start unless the previous stage passed. +Similar to other types of jobs, a delayed job's timer won't start unless the previous stage passed. The following example creates a job named `timed rollout 10%` that is executed 30 minutes after the previous stage has completed: @@ -1475,7 +1987,7 @@ timed rollout 10%: start_in: 30 minutes ``` -You can stop the active timer of a delayed job by clicking the **Unschedule** button. +You can stop the active timer of a delayed job by clicking the **{time-out}** (**Unschedule**) button. This job will never be executed in the future unless you execute the job manually. You can start a delayed job immediately by clicking the **Play** button. @@ -1485,7 +1997,7 @@ GitLab Runner will pick your job soon and start the job. > - Introduced in GitLab 8.9. > - You can read more about environments and find more examples in the -> [documentation about environments](../environments.md). +> [documentation about environments](../environments/index.md). `environment` is used to define that a job deploys to a specific environment. If `environment` is specified and no environment under that name exists, a new @@ -1511,7 +2023,7 @@ deployment to the `production` environment. > `name` keyword. > - The `name` parameter can use any of the defined CI variables, > including predefined, secure variables and `.gitlab-ci.yml` [`variables`](#variables). -> You however cannot use variables defined under `script`. +> You however can't use variables defined under `script`. The `environment` name can contain: @@ -1529,7 +2041,7 @@ Common names are `qa`, `staging`, and `production`, but you can use whatever name works with your workflow. Instead of defining the name of the environment right after the `environment` -keyword, it is also possible to define it as a separate value. For that, use +keyword, it's also possible to define it as a separate value. For that, use the `name` keyword under `environment`: ```yaml @@ -1547,7 +2059,7 @@ deploy to production: > recommended way now is to define it in `.gitlab-ci.yml`. > - The `url` parameter can use any of the defined CI variables, > including predefined, secure variables and `.gitlab-ci.yml` [`variables`](#variables). -> You however cannot use variables defined under `script`. +> You however can't use variables defined under `script`. This is an optional value that when set, it exposes buttons in various places in GitLab which when clicked take you to the defined URL. @@ -1567,7 +2079,7 @@ deploy to production: #### `environment:on_stop` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6669) in GitLab 8.13. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22191) in GitLab 8.13. > - Starting with GitLab 8.14, when you have an environment that has a stop action > defined, GitLab will automatically trigger a stop action when the associated > branch is deleted. @@ -1580,7 +2092,7 @@ Read the `environment:action` section for an example. #### `environment:action` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6669) in GitLab 8.13. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22191) in GitLab 8.13. The `action` keyword is to be used in conjunction with `on_stop` and is defined in the job that is called to close the environment. @@ -1615,7 +2127,7 @@ GitLab's web interface in order to run. Also in the example, `GIT_STRATEGY` is set to `none` so that GitLab Runner won’t try to check out the code after the branch is deleted when the `stop_review_app` -job is [automatically triggered](../environments.md#automatically-stopping-an-environment). +job is [automatically triggered](../environments/index.md#automatically-stopping-an-environment). NOTE: **Note:** The above example overwrites global variables. If your stop environment job depends @@ -1651,7 +2163,7 @@ When `review_app` job is executed and a review app is created, a life period of the environment is set to `1 day`. For more information, see -[the environments auto-stop documentation](../environments.md#environments-auto-stop) +[the environments auto-stop documentation](../environments/index.md#environments-auto-stop) #### `environment:kubernetes` @@ -1677,7 +2189,7 @@ environment, using the `production` [Kubernetes namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/). For more information, see -[Available settings for `kubernetes`](../environments.md#configuring-kubernetes-deployments). +[Available settings for `kubernetes`](../environments/index.md#configuring-kubernetes-deployments). NOTE: **Note:** Kubernetes configuration is not supported for Kubernetes clusters @@ -1687,11 +2199,11 @@ To follow progress on support for GitLab-managed clusters, see the #### Dynamic environments -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/6323) in GitLab 8.12 and GitLab Runner 1.6. -> - The `$CI_ENVIRONMENT_SLUG` was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7983) in GitLab 8.15. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/21971) in GitLab 8.12 and GitLab Runner 1.6. +> - The `$CI_ENVIRONMENT_SLUG` was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22864) in GitLab 8.15. > - The `name` and `url` parameters can use any of the defined CI variables, > including predefined, secure variables and `.gitlab-ci.yml` [`variables`](#variables). -> You however cannot use variables defined under `script`. +> You however can't use variables defined under `script`. For example: @@ -1735,15 +2247,20 @@ Read how caching works and find out some good practices in the cached between jobs. You can only use paths that are within the local working copy. -If `cache` is defined outside the scope of jobs, it means it is set +If `cache` is defined outside the scope of jobs, it means it's set globally and all jobs will use that definition. #### `cache:paths` Use the `paths` directive to choose which files or directories will be cached. Paths -are relative to the project directory (`$CI_PROJECT_DIR`) and cannot directly link outside it. +are relative to the project directory (`$CI_PROJECT_DIR`) and can't directly link outside it. Wildcards can be used that follow the [glob](https://en.wikipedia.org/wiki/Glob_(programming)) -patterns and [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match). +patterns and: + +- In [GitLab Runner 13.0](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2620) and later, +[`doublestar.Glob`](https://pkg.go.dev/github.com/bmatcuk/doublestar@v1.2.2?tab=doc#Match). +- In GitLab Runner 12.10 and earlier, +[`filepath.Match`](https://pkg.go.dev/path/filepath/#Match). Cache all files in `binaries` that end in `.apk` and the `.config` file: @@ -1795,7 +2312,7 @@ set, is just literal `default` which means everything is shared between pipelines and jobs by default, starting from GitLab 9.0. NOTE: **Note:** -The `cache:key` variable cannot contain the `/` character, or the equivalent +The `cache:key` variable can't contain the `/` character, or the equivalent URI-encoded `%2F`; a value made only of dots (`.`, `%2E`) is also forbidden. For example, to enable per-branch caching: @@ -1841,7 +2358,7 @@ cache: - node_modules ``` -In this example we are creating a cache for Ruby and Node.js dependencies that +In this example we're creating a cache for Ruby and Node.js dependencies that is tied to current versions of the `Gemfile.lock` and `package.json` files. Whenever one of these files changes, a new cache key is computed and a new cache is created. Any future job runs using the same `Gemfile.lock` and `package.json` with `cache:key:files` will @@ -1912,12 +2429,12 @@ rspec: > Introduced in GitLab 9.4. -The default behaviour of a caching job is to download the files at the start of +The default behavior of a caching job is to download the files at the start of execution, and to re-upload them at the end. This allows any changes made by the job to be persisted for future runs, and is known as the `pull-push` cache policy. -If you know the job doesn't alter the cached files, you can skip the upload step +If you know the job does not alter the cached files, you can skip the upload step by setting `policy: pull` in the job specification. Typically, this would be twinned with an ordinary cache job at an earlier stage to ensure the cache is updated from time to time: @@ -1973,7 +2490,7 @@ be available for download in the GitLab UI. #### `artifacts:paths` -Paths are relative to the project directory (`$CI_PROJECT_DIR`) and cannot directly +Paths are relative to the project directory (`$CI_PROJECT_DIR`) and can't directly link outside it. Wildcards can be used that follow the [glob](https://en.wikipedia.org/wiki/Glob_(programming)) patterns and [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match). @@ -2000,7 +2517,7 @@ job: You may want to create artifacts only for tagged releases to avoid filling the build server storage with temporary build artifacts. -Create artifacts only for tags (`default-job` will not create artifacts): +Create artifacts only for tags (`default-job` won't create artifacts): ```yaml default-job: @@ -2037,7 +2554,7 @@ in the [merge request](../../user/project/merge_requests/index.md) UI. For example, to match a single file: -```yml +```yaml test: script: [ 'echo 1' ] artifacts: @@ -2050,7 +2567,7 @@ that points to `file1.txt`. An example that will match an entire directory: -```yml +```yaml test: script: [ 'echo 1' ] artifacts: @@ -2081,7 +2598,7 @@ The default name is `artifacts`, which becomes `artifacts.zip` when downloaded. NOTE: **Note:** If your branch-name contains forward slashes -(e.g. `feature/my-feature`) it is advised to use `$CI_COMMIT_REF_SLUG` +(for example `feature/my-feature`) it's advised to use `$CI_COMMIT_REF_SLUG` instead of `$CI_COMMIT_REF_NAME` for proper naming of the artifact. To create an archive with a name of the current job: @@ -2212,15 +2729,15 @@ After their expiry, artifacts are deleted hourly by default (via a cron job), and are not accessible anymore. The value of `expire_in` is an elapsed time in seconds, unless a unit is -provided. Examples of parsable values: +provided. Examples of valid values: -- '42' -- '3 mins 4 sec' -- '2 hrs 20 min' -- '2h20min' -- '6 mos 1 day' -- '47 yrs 6 mos and 4d' -- '3 weeks and 2 days' +- `42` +- `3 mins 4 sec` +- `2 hrs 20 min` +- `2h20min` +- `6 mos 1 day` +- `47 yrs 6 mos and 4d` +- `3 weeks and 2 days` To expire artifacts 1 week after being uploaded: @@ -2230,187 +2747,35 @@ job: expire_in: 1 week ``` -#### `artifacts:reports` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20390) in GitLab 11.2. Requires GitLab Runner 11.2 and above. - -The `reports` keyword is used for collecting test reports, code quality reports, and security reports from jobs. -It also exposes these reports in GitLab's UI (merge requests, pipeline views, and security dashboards). - -NOTE: **Note:** -The test reports are collected regardless of the job results (success or failure). -You can use [`artifacts:expire_in`](#artifactsexpire_in) to set up an expiration -date for their artifacts. - -NOTE: **Note:** -If you also want the ability to browse the report output files, include the -[`artifacts:paths`](#artifactspaths) keyword. - -##### `artifacts:reports:junit` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20390) in GitLab 11.2. Requires GitLab Runner 11.2 and above. - -The `junit` report collects [JUnit XML files](https://www.ibm.com/support/knowledgecenter/en/SSQ2R2_14.1.0/com.ibm.rsar.analysis.codereview.cobol.doc/topics/cac_useresults_junit.html) -as artifacts. Although JUnit was originally developed in Java, there are many -[third party ports](https://en.wikipedia.org/wiki/JUnit#Ports) for other -languages like JavaScript, Python, Ruby, etc. - -See [JUnit test reports](../junit_test_reports.md) for more details and examples. -Below is an example of collecting a JUnit XML file from Ruby's RSpec test tool: - -```yaml -rspec: - stage: test - script: - - bundle install - - rspec --format RspecJunitFormatter --out rspec.xml - artifacts: - reports: - junit: rspec.xml -``` - -The collected JUnit reports will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests. - NOTE: **Note:** -In case the JUnit tool you use exports to multiple XML files, you can specify -multiple test report paths within a single job and they will be automatically -concatenated into a single file. Use a filename pattern (`junit: rspec-*.xml`), -an array of filenames (`junit: [rspec-1.xml, rspec-2.xml, rspec-3.xml]`), or a -combination thereof (`junit: [rspec.xml, test-results/TEST-*.xml]`). - -##### `artifacts:reports:dotenv` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. Requires GitLab Runner 11.5 and later. - -The `dotenv` report collects a set of environment variables as artifacts. - -The collected variables are registered as runtime-created variables of the job, -which is useful to [set dynamic environment URLs after a job finishes](../environments.md#set-dynamic-environment-urls-after-a-job-finishes). -It is not available for download through the web interface. - -There are a couple of limitations on top of the [original dotenv rules](https://github.com/motdotla/dotenv#rules). - -- The variable key can contain only letters, digits and underscore ('_'). -- The size of dotenv file must be smaller than 5 kilobytes. -- The number of variables must be less than 10. -- It doesn't support variable substitution in the dotenv file itself. -- It doesn't support empty lines and comments (`#`) in dotenv file. -- It doesn't support quote escape, spaces in a quote, a new line expansion in a quote, in dotenv file. - -##### `artifacts:reports:cobertura` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3708) in GitLab 12.9. Requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 and above. - -The `cobertura` report collects [Cobertura coverage XML files](../../user/project/merge_requests/test_coverage_visualization.md). -The collected Cobertura coverage reports will be uploaded to GitLab as an artifact -and will be automatically shown in merge requests. - -Cobertura was originally developed for Java, but there are many -third party ports for other languages like JavaScript, Python, Ruby, etc. - -##### `artifacts:reports:codequality` **(STARTER)** - -> Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. - -The `codequality` report collects [CodeQuality issues](../../user/project/merge_requests/code_quality.md) -as artifacts. - -The collected Code Quality report will be uploaded to GitLab as an artifact and will -be summarized in merge requests. It is not available for download through the web interface. - -##### `artifacts:reports:sast` **(ULTIMATE)** - -> Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. - -The `sast` report collects [SAST vulnerabilities](../../user/application_security/sast/index.md) -as artifacts. - -The collected SAST report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It is also used to provide data for security -dashboards. It is not available for download through the web interface. - -##### `artifacts:reports:dependency_scanning` **(ULTIMATE)** - -> Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. - -The `dependency_scanning` report collects [Dependency Scanning vulnerabilities](../../user/application_security/dependency_scanning/index.md) -as artifacts. - -The collected Dependency Scanning report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It is also used to provide data for security -dashboards. It is not available for download through the web interface. - -##### `artifacts:reports:container_scanning` **(ULTIMATE)** - -> Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. - -The `container_scanning` report collects [Container Scanning vulnerabilities](../../user/application_security/container_scanning/index.md) -as artifacts. - -The collected Container Scanning report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It is also used to provide data for security -dashboards. It is not available for download through the web interface. - -##### `artifacts:reports:dast` **(ULTIMATE)** - -> Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. - -The `dast` report collects [DAST vulnerabilities](../../user/application_security/dast/index.md) -as artifacts. - -The collected DAST report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It is also used to provide data for security -dashboards. It is not available for download through the web interface. - -##### `artifacts:reports:license_management` **(ULTIMATE)** - -CAUTION: **Warning:** -This artifact is still valid but was **deprecated** in favor of the -[artifacts:reports:license_scanning](#artifactsreportslicense_scanning-ultimate) -introduced in GitLab 12.8. - -> Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. - -The `license_management` report collects [Licenses](../../user/compliance/license_compliance/index.md) -as artifacts. - -The collected License Compliance report will be uploaded to GitLab as an artifact and will -be summarized in the merge requests and pipeline view. It is also used to provide data for security -dashboards. It is not available for download through the web interface. - -##### `artifacts:reports:license_scanning` **(ULTIMATE)** - -> Introduced in GitLab 12.8. Requires GitLab Runner 11.5 and above. +For artifacts created in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/16267) +and later, the latest artifact for a ref is always kept, regardless of the expiry time. -The `license_scanning` report collects [Licenses](../../user/compliance/license_compliance/index.md) -as artifacts. - -The License Compliance report will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests, pipeline view and provide data for security -dashboards. - -##### `artifacts:reports:performance` **(PREMIUM)** - -> Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. - -The `performance` report collects [Performance metrics](../../user/project/merge_requests/browser_performance_testing.md) -as artifacts. - -The collected Performance report will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests. It is not available for download through the web interface. - -##### `artifacts:reports:metrics` **(PREMIUM)** - -> Introduced in GitLab 11.10. - -The `metrics` report collects [Metrics](../../ci/metrics_reports.md) -as artifacts. +#### `artifacts:reports` -The collected Metrics report will be uploaded to GitLab as an artifact and will -be automatically shown in merge requests. It is not available for download through the web interface. +The [`artifacts:reports` keyword](../pipelines/job_artifacts.md#artifactsreports) +is used for collecting test reports, code quality reports, and security reports from jobs. +It also exposes these reports in GitLab's UI (merge requests, pipeline views, and security dashboards). -### `dependencies` +These are the available report types: + +| Parameter | Description | +|--------------------------------------------------------------------------------------------------------------------------------------|-------------| +| [`artifacts:reports:junit`](../pipelines/job_artifacts.md#artifactsreportsjunit) | The `junit` report collects JUnit XML files. | +| [`artifacts:reports:dotenv`](../pipelines/job_artifacts.md#artifactsreportsdotenv) | The `dotenv` report collects a set of environment variables. | +| [`artifacts:reports:cobertura`](../pipelines/job_artifacts.md#artifactsreportscobertura) | The `cobertura` report collects Cobertura coverage XML files. | +| [`artifacts:reports:terraform`](../pipelines/job_artifacts.md#artifactsreportsterraform) | The `terraform` report collects Terraform `tfplan.json` files. | +| [`artifacts:reports:codequality`](../pipelines/job_artifacts.md#artifactsreportscodequality-starter) **(STARTER)** | The `codequality` report collects CodeQuality issues. | +| [`artifacts:reports:sast`](../pipelines/job_artifacts.md#artifactsreportssast-ultimate) **(ULTIMATE)** | The `sast` report collects Static Application Security Testing vulnerabilities. | +| [`artifacts:reports:dependency_scanning`](../pipelines/job_artifacts.md#artifactsreportsdependency_scanning-ultimate) **(ULTIMATE)** | The `dependency_scanning` report collects Dependency Scanning vulnerabilities. | +| [`artifacts:reports:container_scanning`](../pipelines/job_artifacts.md#artifactsreportscontainer_scanning-ultimate) **(ULTIMATE)** | The `container_scanning` report collects Container Scanning vulnerabilities. | +| [`artifacts:reports:dast`](../pipelines/job_artifacts.md#artifactsreportsdast-ultimate) **(ULTIMATE)** | The `dast` report collects Dynamic Application Security Testing vulnerabilities. | +| [`artifacts:reports:license_management`](../pipelines/job_artifacts.md#artifactsreportslicense_management-ultimate) **(ULTIMATE)** | The `license_management` report collects Licenses (*removed from GitLab 13.0*). | +| [`artifacts:reports:license_scanning`](../pipelines/job_artifacts.md#artifactsreportslicense_scanning-ultimate) **(ULTIMATE)** | The `license_scanning` report collects Licenses. | +| [`artifacts:reports:performance`](../pipelines/job_artifacts.md#artifactsreportsperformance-premium) **(PREMIUM)** | The `performance` report collects Performance metrics. | +| [`artifacts:reports:metrics`](../pipelines/job_artifacts.md#artifactsreportsmetrics-premium) **(PREMIUM)** | The `metrics` report collects Metrics. | + +#### `dependencies` > Introduced in GitLab 8.6 and GitLab Runner v1.1.1. @@ -2424,7 +2789,7 @@ You can only define jobs from stages that are executed before the current one. An error will be shown if you define jobs from the current stage or next ones. Defining an empty array will skip downloading any artifacts for that job. The status of the previous job is not considered when using `dependencies`, so -if it failed or it is a manual job that was not run, no error occurs. +if it failed or it's a manual job that was not run, no error occurs. In the following example, we define two jobs with artifacts, `build:osx` and `build:linux`. When the `test:osx` is executed, the artifacts from `build:osx` @@ -2466,7 +2831,7 @@ deploy: script: make deploy ``` -#### When a dependent job will fail +##### When a dependent job will fail > Introduced in GitLab 10.3. @@ -2480,193 +2845,9 @@ You can ask your administrator to [flip this switch](../../administration/job_artifacts.md#validation-for-dependencies) and bring back the old behavior. -### `needs` - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47063) in GitLab 12.2. -> - In GitLab 12.3, maximum number of jobs in `needs` array raised from five to 50. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30631) in GitLab 12.8, `needs: []` lets jobs start immediately. - -The `needs:` keyword enables executing jobs out-of-order, allowing you to implement -a [directed acyclic graph](../directed_acyclic_graph/index.md) in your `.gitlab-ci.yml`. - -This lets you run some jobs without waiting for other ones, disregarding stage ordering -so you can have multiple stages running concurrently. - -Let's consider the following example: - -```yaml -linux:build: - stage: build - -mac:build: - stage: build - -lint: - stage: test - needs: [] - -linux:rspec: - stage: test - needs: ["linux:build"] - -linux:rubocop: - stage: test - needs: ["linux:build"] - -mac:rspec: - stage: test - needs: ["mac:build"] - -mac:rubocop: - stage: test - needs: ["mac:build"] - -production: - stage: deploy -``` - -This example creates four paths of execution: - -- Linter: the `lint` job will run immediately without waiting for the `build` stage to complete because it has no needs (`needs: []`). - -- Linux path: the `linux:rspec` and `linux:rubocop` jobs will be run as soon - as the `linux:build` job finishes without waiting for `mac:build` to finish. - -- macOS path: the `mac:rspec` and `mac:rubocop` jobs will be run as soon - as the `mac:build` job finishes, without waiting for `linux:build` to finish. - -- The `production` job will be executed as soon as all previous jobs - finish; in this case: `linux:build`, `linux:rspec`, `linux:rubocop`, - `mac:build`, `mac:rspec`, `mac:rubocop`. - -#### Requirements and limitations - -- If `needs:` is set to point to a job that is not instantiated - because of `only/except` rules or otherwise does not exist, the - pipeline will be created with YAML error. -- We are temporarily limiting the maximum number of jobs that a single job can - need in the `needs:` array: - - For GitLab.com, the limit is ten. For more information, see our - [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/7541). - - For self-managed instances, the limit is: - - 10, if the `ci_dag_limit_needs` feature flag is enabled (default). - - 50, if the `ci_dag_limit_needs` feature flag is disabled. -- If `needs:` refers to a job that is marked as `parallel:`. - the current job will depend on all parallel jobs created. -- `needs:` is similar to `dependencies:` in that it needs to use jobs from prior stages, - meaning it is impossible to create circular dependencies. Depending on jobs in the - current stage is not possible either, but support [is planned](https://gitlab.com/gitlab-org/gitlab/issues/30632). -- Related to the above, stages must be explicitly defined for all jobs - that have the keyword `needs:` or are referred to by one. - -##### Changing the `needs:` job limit - -The maximum number of jobs that can be defined within `needs:` defaults to 10, but -can be changed to 50 via a feature flag. To change the limit to 50, -[start a Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) -and run: - -```ruby -Feature::disable(:ci_dag_limit_needs) -``` - -To set it back to 10, run the opposite command: - -```ruby -Feature::enable(:ci_dag_limit_needs) -``` - -#### Artifact downloads with `needs` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14311) in GitLab v12.6. - -When using `needs`, artifact downloads are controlled with `artifacts: true` or `artifacts: false`. -The `dependencies` keyword should not be used with `needs`, as this is deprecated since GitLab 12.6. - -In the example below, the `rspec` job will download the `build_job` artifacts, while the -`rubocop` job will not: - -```yaml -build_job: - stage: build - artifacts: - paths: - - binaries/ - -rspec: - stage: test - needs: - - job: build_job - artifacts: true - -rubocop: - stage: test - needs: - - job: build_job - artifacts: false -``` - -Additionally, in the three syntax examples below, the `rspec` job will download the artifacts -from all three `build_jobs`, as `artifacts` is true for `build_job_1`, and will -**default** to true for both `build_job_2` and `build_job_3`. - -```yaml -rspec: - needs: - - job: build_job_1 - artifacts: true - - job: build_job_2 - - build_job_3 -``` - -#### Cross project artifact downloads with `needs` **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14311) in GitLab v12.7. - -`needs` can be used to download artifacts from up to five jobs in pipelines on -[other refs in the same project](#artifact-downloads-between-pipelines-in-the-same-project), -or pipelines in different projects: - -```yaml -build_job: - stage: build - script: - - ls -lhR - needs: - - project: group/project-name - job: build-1 - ref: master - artifacts: true -``` - -`build_job` will download the artifacts from the latest successful `build-1` job -on the `master` branch in the `group/project-name` project. - -##### Artifact downloads between pipelines in the same project - -`needs` can be used to download artifacts from different pipelines in the current project -by setting the `project` keyword as the current project's name, and specifying a ref. -In the example below, `build_job` will download the artifacts for the latest successful -`build-1` job with the `other-ref` ref: - -```yaml -build_job: - stage: build - script: - - ls -lhR - needs: - - project: group/same-project-name - job: build-1 - ref: other-ref - artifacts: true -``` - -NOTE: **Note:** -Downloading artifacts from jobs that are run in [`parallel:`](#parallel) is not supported. - ### `coverage` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7447) in GitLab 8.17. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/20428) in GitLab 8.17. `coverage` allows you to configure how code coverage will be extracted from the job output. @@ -2686,13 +2867,13 @@ job1: ### `retry` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12909) in GitLab 9.5. -> - [Behaviour expanded](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21758) in GitLab 11.5 to control on which failures to retry. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3442) in GitLab 9.5. +> - [Behavior expanded](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3515) in GitLab 11.5 to control on which failures to retry. `retry` allows you to configure how many times a job is going to be retried in case of a failure. -When a job fails and has `retry` configured, it is going to be processed again +When a job fails and has `retry` configured, it's going to be processed again up to the amount of times specified by the `retry` keyword. If `retry` is set to 2, and a job succeeds in a second run (first retry), it won't be retried @@ -2744,7 +2925,7 @@ Possible values for `when` are: Please make sure to update `RETRY_WHEN_IN_DOCUMENTATION` array in `spec/lib/gitlab/ci/config/entry/retry_spec.rb` if you change any of the documented values below. The test there makes sure that all documented - values are really valid as a config option and therefore should always + values are really valid as a configuration option and therefore should always stay in sync with this documentation. --> @@ -2753,16 +2934,18 @@ Possible values for `when` are: - `script_failure`: Retry when the script failed. - `api_failure`: Retry on API failure. - `stuck_or_timeout_failure`: Retry when the job got stuck or timed out. -- `runner_system_failure`: Retry if there was a runner system failure (e.g. setting up the job failed). +- `runner_system_failure`: Retry if there was a runner system failure (for example, job setup failed). - `missing_dependency_failure`: Retry if a dependency was missing. - `runner_unsupported`: Retry if the runner was unsupported. - `stale_schedule`: Retry if a delayed job could not be executed. - `job_execution_timeout`: Retry if the script exceeded the maximum execution time set for the job. -- `archived_failure`: Retry if the job is archived and cannot be run. +- `archived_failure`: Retry if the job is archived and can't be run. - `unmet_prerequisites`: Retry if the job failed to complete prerequisite tasks. - `scheduler_failure`: Retry if the scheduler failed to assign the job to a runner. - `data_integrity_failure`: Retry if there was a structural integrity problem detected. +You can specify the number of [retry attempts for certain stages of job execution](#job-stages-attempts) using variables. + ### `timeout` > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14887) in GitLab 12.3. @@ -2780,17 +2963,17 @@ test: ``` The job-level timeout can exceed the -[project-level timeout](../pipelines/settings.md#timeout) but can not +[project-level timeout](../pipelines/settings.md#timeout) but can't exceed the Runner-specific timeout. ### `parallel` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22631) in GitLab 11.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/21480) in GitLab 11.5. `parallel` allows you to configure how many instances of a job to run in parallel. This value has to be greater than or equal to two (2) and less than or equal to 50. -This creates N instances of the same job that run in parallel. They're named +This creates N instances of the same job that run in parallel. They are named sequentially from `job_name 1/N` to `job_name N/N`. For every job, `CI_NODE_INDEX` and `CI_NODE_TOTAL` [environment variables](../variables/README.md#predefined-environment-variables) are set. @@ -2868,7 +3051,7 @@ staging: #### Complex `trigger` syntax for multi-project pipelines -It is possible to configure a branch name that GitLab will use to create +It's possible to configure a branch name that GitLab will use to create a downstream pipeline with: ```yaml @@ -2883,7 +3066,7 @@ staging: branch: stable ``` -It is possible to mirror the status from a triggered pipeline: +It's possible to mirror the status from a triggered pipeline: ```yaml trigger_job: @@ -2892,7 +3075,7 @@ trigger_job: strategy: depend ``` -It is possible to mirror the status from an upstream pipeline: +It's possible to mirror the status from an upstream pipeline: ```yaml upstream_bridge: @@ -2915,7 +3098,7 @@ trigger_job: ``` Similar to [multi-project pipelines](../multi_project_pipelines.md#mirroring-status-from-triggered-pipeline), -it is possible to mirror the status from a triggered pipeline: +it's possible to mirror the status from a triggered pipeline: ```yaml trigger_job: @@ -2971,9 +3154,18 @@ This can help keep your pipeline execution linear. In the example above, jobs fr subsequent stages will wait for the triggered pipeline to successfully complete before starting, at the cost of reduced parallelization. +#### Trigger a pipeline by API call + +Triggers can be used to force a rebuild of a specific branch, tag or commit, +with an API call when a pipeline gets created using a trigger token. + +Not to be confused with the [`trigger`](#trigger) parameter. + +[Read more in the triggers documentation.](../triggers/README.md) + ### `interruptible` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23464) in GitLab 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32022) in GitLab 12.3. `interruptible` is used to indicate that a job should be canceled if made redundant by a newer pipeline run. Defaults to `false`. This value will only be used if the [automatic cancellation of redundant pipelines feature](../pipelines/settings.md#auto-cancel-pending-pipelines) @@ -2981,8 +3173,8 @@ is enabled. When enabled, a pipeline on the same branch will be canceled when: -- It is made redundant by a newer pipeline run. -- Either all jobs are set as interruptible, or any uninterruptible jobs have not started. +- it's made redundant by a newer pipeline run. +- Either all jobs are set as interruptible, or any uninterruptible jobs haven't started. Pending jobs are always considered interruptible. @@ -3031,7 +3223,7 @@ Sometimes running multiples jobs or pipelines at the same time in an environment can lead to errors during the deployment. To avoid these errors, the `resource_group` attribute can be used to ensure that -the Runner will not run certain jobs simultaneously. +the Runner won't run certain jobs simultaneously. When the `resource_group` key is defined for a job in `.gitlab-ci.yml`, job executions are mutually exclusive across different pipelines for the same project. @@ -3048,7 +3240,7 @@ deploy-to-production: ``` In this case, if a `deploy-to-production` job is running in a pipeline, and a new -`deploy-to-production` job is created in a different pipeline, it will not run until +`deploy-to-production` job is created in a different pipeline, it won't run until the currently running/pending `deploy-to-production` job is finished. As a result, you can ensure that concurrent deployments will never happen to the production environment. @@ -3057,514 +3249,8 @@ is when deploying to physical devices. You may have more than one physical devic one can be deployed to, but there can be only one deployment per device at any given time. NOTE: **Note:** -This key can only contain letters, digits, `-`, `_`, `/`, `$`, `{`, `}`, `.`, and spaces, but it cannot start or end with `/`. - -### `include` - -> - Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. -> - Available for Starter, Premium and Ultimate since 10.6. -> - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21603) to GitLab Core in 11.4. - -Using the `include` keyword, you can allow the inclusion of external YAML files. -`include` requires the external YAML file to have the extensions `.yml` or `.yaml`, -otherwise the external file will not be included. - -The files defined in `include` are: - -- Deep merged with those in `.gitlab-ci.yml`. -- Always evaluated first and merged with the content of `.gitlab-ci.yml`, - regardless of the position of the `include` keyword. - -TIP: **Tip:** -Use merging to customize and override included CI/CD configurations with local -definitions. - -NOTE: **Note:** -Using YAML aliases across different YAML files sourced by `include` is not -supported. You must only refer to aliases in the same file. Instead -of using YAML anchors, you can use the [`extends` keyword](#extends). - -`include` supports four include methods: - -- [`local`](#includelocal) -- [`file`](#includefile) -- [`template`](#includetemplate) -- [`remote`](#includeremote) - -See [usage examples](#include-examples). - -NOTE: **Note:** -`.gitlab-ci.yml` configuration included by all methods is evaluated at pipeline creation. -The configuration is a snapshot in time and persisted in the database. Any changes to -referenced `.gitlab-ci.yml` configuration will not be reflected in GitLab until the next pipeline is created. - -#### `include:local` - -`include:local` includes a file from the same repository as `.gitlab-ci.yml`. -It's referenced using full paths relative to the root directory (`/`). - -You can only use files that are currently tracked by Git on the same branch -your configuration file is on. In other words, when using a `include:local`, make -sure that both `.gitlab-ci.yml` and the local file are on the same branch. - -All [nested includes](#nested-includes) will be executed in the scope of the same project, -so it is possible to use local, project, remote, or template includes. - -NOTE: **Note:** -Including local files through Git submodules paths is not supported. - -Example: - -```yaml -include: - - local: '/templates/.gitlab-ci-template.yml' -``` - -#### `include:file` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53903) in GitLab 11.7. - -To include files from another private project under the same GitLab instance, -use `include:file`. This file is referenced using full paths relative to the -root directory (`/`). For example: - -```yaml -include: - - project: 'my-group/my-project' - file: '/templates/.gitlab-ci-template.yml' -``` - -You can also specify `ref`, with the default being the `HEAD` of the project: - -```yaml -include: - - project: 'my-group/my-project' - ref: master - file: '/templates/.gitlab-ci-template.yml' - - - project: 'my-group/my-project' - ref: v1.0.0 - file: '/templates/.gitlab-ci-template.yml' - - - project: 'my-group/my-project' - ref: 787123b47f14b552955ca2786bc9542ae66fee5b # Git SHA - file: '/templates/.gitlab-ci-template.yml' -``` - -All [nested includes](#nested-includes) will be executed in the scope of the target project, -so it is possible to use local (relative to target project), project, remote -or template includes. - -#### `include:template` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53445) in GitLab 11.7. - -`include:template` can be used to include `.gitlab-ci.yml` templates that are -[shipped with GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates). - -For example: - -```yaml -# File sourced from GitLab's template collection -include: - - template: Auto-DevOps.gitlab-ci.yml -``` - -Multiple `include:template` files: - -```yaml -include: - - template: Android-Fastlane.gitlab-ci.yml - - template: Auto-DevOps.gitlab-ci.yml -``` - -All [nested includes](#nested-includes) will be executed only with the permission of the user, -so it is possible to use project, remote or template includes. - -#### `include:remote` - -`include:remote` can be used to include a file from a different location, -using HTTP/HTTPS, referenced by using the full URL. The remote file must be -publicly accessible through a simple GET request as authentication schemas -in the remote URL is not supported. For example: - -```yaml -include: - - remote: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-template.yml' -``` - -All nested includes will be executed without context as public user, so only another remote, -or public project, or template is allowed. - -#### Nested includes - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/56836) in GitLab 11.9. - -Nested includes allow you to compose a set of includes. -A total of 100 includes is allowed. -Duplicate includes are considered a configuration error. - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/28212) in GitLab 12.4. - -A hard limit of 30 seconds was set for resolving all files. - -#### `include` examples - -Here are a few more `include` examples. - -##### Single string or array of multiple values - -You can include your extra YAML file(s) either as a single string or -an array of multiple values. The following examples are all valid. - -Single string with the `include:local` method implied: - -```yaml -include: '/templates/.after-script-template.yml' -``` - -Array with `include` method implied: - -```yaml -include: - - 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' - - '/templates/.after-script-template.yml' -``` - -Single string with `include` method specified explicitly: - -```yaml -include: - remote: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' -``` - -Array with `include:remote` being the single item: - -```yaml -include: - - remote: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' -``` - -Array with multiple `include` methods specified explicitly: - -```yaml -include: - - remote: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' - - local: '/templates/.after-script-template.yml' - - template: Auto-DevOps.gitlab-ci.yml -``` - -Array mixed syntax: - -```yaml -include: - - 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' - - '/templates/.after-script-template.yml' - - template: Auto-DevOps.gitlab-ci.yml - - project: 'my-group/my-project' - ref: master - file: '/templates/.gitlab-ci-template.yml' -``` - -##### Re-using a `before_script` template - -In the following example, the content of `.before-script-template.yml` will be -automatically fetched and evaluated along with the content of `.gitlab-ci.yml`. - -Content of `https://gitlab.com/awesome-project/raw/master/.before-script-template.yml`: - -```yaml -before_script: - - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs - - gem install bundler --no-document - - bundle install --jobs $(nproc) "${FLAGS[@]}" -``` - -Content of `.gitlab-ci.yml`: - -```yaml -include: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' - -rspec: - script: - - bundle exec rspec -``` - -##### Overriding external template values - -The following example shows specific YAML-defined variables and details of the -`production` job from an include file being customized in `.gitlab-ci.yml`. - -Content of `https://company.com/autodevops-template.yml`: - -```yaml -variables: - POSTGRES_USER: user - POSTGRES_PASSWORD: testing_password - POSTGRES_DB: $CI_ENVIRONMENT_SLUG - -production: - stage: production - script: - - install_dependencies - - deploy - environment: - name: production - url: https://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN - only: - - master -``` - -Content of `.gitlab-ci.yml`: - -```yaml -include: 'https://company.com/autodevops-template.yml' - -image: alpine:latest - -variables: - POSTGRES_USER: root - POSTGRES_PASSWORD: secure_password - -stages: - - build - - test - - production - -production: - environment: - url: https://domain.com -``` - -In this case, the variables `POSTGRES_USER` and `POSTGRES_PASSWORD` along -with the environment url of the `production` job defined in -`autodevops-template.yml` have been overridden by new values defined in -`.gitlab-ci.yml`. - -The merging lets you extend and override dictionary mappings, but -you cannot add or modify items to an included array. For example, to add -an additional item to the production job script, you must repeat the -existing script items: - -Content of `https://company.com/autodevops-template.yml`: - -```yaml -production: - stage: production - script: - - install_dependencies - - deploy -``` - -Content of `.gitlab-ci.yml`: - -```yaml -include: 'https://company.com/autodevops-template.yml' - -stages: - - production - -production: - script: - - install_dependencies - - deploy - - notify_owner -``` - -In this case, if `install_dependencies` and `deploy` were not repeated in -`.gitlab-ci.yml`, they would not be part of the script for the `production` -job in the combined CI configuration. - -##### Using nested includes - -The examples below show how includes can be nested from different sources -using a combination of different methods. - -In this example, `.gitlab-ci.yml` includes local the file `/.gitlab-ci/another-config.yml`: - -```yaml -include: - - local: /.gitlab-ci/another-config.yml -``` - -The `/.gitlab-ci/another-config.yml` includes a template and the `/templates/docker-workflow.yml` file -from another project: - -```yaml -include: - - template: Bash.gitlab-ci.yml - - project: group/my-project - file: /templates/docker-workflow.yml -``` - -The `/templates/docker-workflow.yml` present in `group/my-project` includes two local files -of the `group/my-project`: - -```yaml -include: - - local: /templates/docker-build.yml - - local: /templates/docker-testing.yml -``` - -Our `/templates/docker-build.yml` present in `group/my-project` adds a `docker-build` job: - -```yaml -docker-build: - script: docker build -t my-image . -``` - -Our second `/templates/docker-test.yml` present in `group/my-project` adds a `docker-test` job: - -```yaml -docker-test: - script: docker run my-image /run/tests.sh -``` - -### `extends` - -> Introduced in GitLab 11.3. - -`extends` defines entry names that a job that uses `extends` is going to -inherit from. - -It is an alternative to using [YAML anchors](#anchors) and is a little -more flexible and readable: - -```yaml -.tests: - script: rake test - stage: test - only: - refs: - - branches - -rspec: - extends: .tests - script: rake rspec - only: - variables: - - $RSPEC -``` - -In the example above, the `rspec` job inherits from the `.tests` template job. -GitLab will perform a reverse deep merge based on the keys. GitLab will: - -- Merge the `rspec` contents into `.tests` recursively. -- Not merge the values of the keys. - -This results in the following `rspec` job: - -```yaml -rspec: - script: rake rspec - stage: test - only: - refs: - - branches - variables: - - $RSPEC -``` - -NOTE: **Note:** -Note that `script: rake test` has been overwritten by `script: rake rspec`. - -If you do want to include the `rake test`, see [`before_script` and `after_script`](#before_script-and-after_script). - -`.tests` in this example is a [hidden key](#hidden-keys-jobs), but it's -possible to inherit from regular jobs as well. - -`extends` supports multi-level inheritance, however it is not recommended to -use more than three levels. The maximum nesting level that is supported is 10. -The following example has two levels of inheritance: - -```yaml -.tests: - only: - - pushes - -.rspec: - extends: .tests - script: rake rspec - -rspec 1: - variables: - RSPEC_SUITE: '1' - extends: .rspec - -rspec 2: - variables: - RSPEC_SUITE: '2' - extends: .rspec - -spinach: - extends: .tests - script: rake spinach -``` - -In GitLab 12.0 and later, it's also possible to use multiple parents for -`extends`. The algorithm used for merge is "closest scope wins", so -keys from the last member will always shadow anything defined on other -levels. For example: - -```yaml -.only-important: - only: - - master - - stable - tags: - - production - -.in-docker: - tags: - - docker - image: alpine - -rspec: - extends: - - .only-important - - .in-docker - script: - - rake rspec -``` - -This results in the following `rspec` job: - -```yaml -rspec: - only: - - master - - stable - tags: - - docker - image: alpine - script: - - rake rspec -``` - -### Using `extends` and `include` together - -`extends` works across configuration files combined with `include`. - -For example, if you have a local `included.yml` file: - -```yaml -.template: - script: - - echo Hello! -``` - -Then, in `.gitlab-ci.yml` you can use it like this: - -```yaml -include: included.yml - -useTemplate: - image: alpine - extends: .template -``` - -This will run a job called `useTemplate` that runs `echo Hello!` as defined in -the `.template` job, and uses the `alpine` Docker image as defined in the local job. +This key can only contain letters, digits, `-`, `_`, `/`, `$`, `{`, `}`, `.`, and spaces. +It can't start or end with `/`. ### `pages` @@ -3576,7 +3262,7 @@ requirements below must be met: - `artifacts` with a path to the `public/` directory must be defined. The example below simply moves all files from the root of the project to the -`public/` directory. The `.public` workaround is so `cp` doesn't also copy +`public/` directory. The `.public` workaround is so `cp` does not also copy `public/` to itself in an infinite loop: ```yaml @@ -3595,13 +3281,13 @@ pages: Read more on [GitLab Pages user documentation](../../user/project/pages/index.md). -### `variables` +## `variables` > Introduced in GitLab Runner v0.5.0. NOTE: **Note:** Integers (as well as strings) are legal both for variable's name and value. -Floats are not legal and cannot be used. +Floats are not legal and can't be used. GitLab CI/CD allows you to define variables inside `.gitlab-ci.yml` that are then passed in the job environment. They can be set globally and per-job. @@ -3632,7 +3318,7 @@ which can be set in GitLab's UI. Learn more about [variables and their priority](../variables/README.md). -#### Git strategy +### Git strategy > - Introduced in GitLab 8.9 as an experimental feature. > - `GIT_STRATEGY=none` requires GitLab Runner v1.7+. @@ -3655,7 +3341,7 @@ variables: ``` `fetch` is faster as it re-uses the local working copy (falling back to `clone` -if it doesn't exist). `git clean` is used to undo any changes made by the last +if it does not exist). `git clean` is used to undo any changes made by the last job, and `git fetch` is used to retrieve commits made since the last job ran. ```yaml @@ -3664,9 +3350,9 @@ variables: ``` `none` also re-uses the local working copy, but skips all Git operations -(including GitLab Runner's pre-clone script, if present). It is mostly useful -for jobs that operate exclusively on artifacts (e.g., `deploy`). Git repository -data may be present, but it is certain to be out of date, so you should only +(including GitLab Runner's pre-clone script, if present). It's mostly useful +for jobs that operate exclusively on artifacts (for examples `deploy`). Git repository +data may be present, but it's certain to be out of date, so you should only rely on files brought into the local working copy from cache or artifacts. ```yaml @@ -3679,7 +3365,7 @@ NOTE: **Note:** `GIT_STRATEGY` is not supported for but may be in the future. See the [support Git strategy with Kubernetes executor feature proposal](https://gitlab.com/gitlab-org/gitlab-runner/issues/3847) for updates. -#### Git submodule strategy +### Git submodule strategy > Requires GitLab Runner v1.10+. @@ -3689,10 +3375,10 @@ globally or per-job in the [`variables`](#variables) section. There are three possible values: `none`, `normal`, and `recursive`: -- `none` means that submodules will not be included when fetching the project +- `none` means that submodules won't be included when fetching the project code. This is the default, which matches the pre-v1.10 behavior. -- `normal` means that only the top-level submodules will be included. It is +- `normal` means that only the top-level submodules will be included. It's equivalent to: ```shell @@ -3703,7 +3389,7 @@ There are three possible values: `none`, `normal`, and `recursive`: - `recursive` means that all submodules (including submodules of submodules) will be included. This feature needs Git v1.8.1 and later. When using a GitLab Runner with an executor not based on Docker, make sure the Git version - meets that requirement. It is equivalent to: + meets that requirement. It's equivalent to: ```shell git submodule sync --recursive @@ -3717,7 +3403,7 @@ Note that for this feature to work correctly, the submodules must be configured - a relative path to another repository on the same GitLab server. See the [Git submodules](../git_submodules.md) documentation. -#### Git checkout +### Git checkout > Introduced in GitLab Runner 9.3. @@ -3746,7 +3432,7 @@ script: - git merge $CI_COMMIT_SHA ``` -#### Git clean flags +### Git clean flags > Introduced in GitLab Runner 11.10 @@ -3773,7 +3459,7 @@ script: - ls -al cache/ ``` -#### Job stages attempts +### Job stages attempts > Introduced in GitLab, it requires GitLab Runner v1.9+. @@ -3798,7 +3484,7 @@ variables: You can set them globally or per-job in the [`variables`](#variables) section. -#### Shallow cloning +### Shallow cloning > Introduced in GitLab 8.9 as an experimental feature. @@ -3816,7 +3502,7 @@ jobs, jobs may fail. Since Git fetching and cloning is based on a ref, such as a branch name, Runners can't clone a specific commit SHA. If there are multiple jobs in the queue, or -you are retrying an old job, the commit to be tested needs to be within the +you're retrying an old job, the commit to be tested needs to be within the Git history that is cloned. Setting too small a value for `GIT_DEPTH` can make it impossible to run these old commits. You will see `unresolved reference` in job logs. You should then reconsider changing `GIT_DEPTH` to a higher value. @@ -3833,46 +3519,9 @@ variables: You can set it globally or per-job in the [`variables`](#variables) section. -## Deprecated parameters - -The following parameters are deprecated. - -### Globally-defined `types` - -CAUTION: **Deprecated:** -`types` is deprecated, and could be removed in a future release. -Use [`stages`](#stages) instead. - -### Job-defined `type` - -CAUTION: **Deprecated:** -`type` is deprecated, and could be removed in one of the future releases. -Use [`stage`](#stage) instead. - -### Globally-defined `image`, `services`, `cache`, `before_script`, `after_script` - -Defining `image`, `services`, `cache`, `before_script`, and -`after_script` globally is deprecated. Support could be removed -from a future release. - -Use [`default:`](#setting-default-parameters) instead. For example: - -```yaml -default: - image: ruby:2.5 - services: - - docker:dind - cache: - paths: [vendor/] - before_script: - - bundle install --path vendor/ - after_script: - - rm -rf tmp/ -``` - -## Custom build directories +### Custom build directories -> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1267) in GitLab Runner 11.10 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2211) in GitLab Runner 11.10 NOTE: **Note:** This can only be used when `custom_build_dir` is enabled in the [Runner's @@ -3885,7 +3534,7 @@ specific directory (Go projects, for example). In that case, you can specify the `GIT_CLONE_PATH` variable to tell the Runner in which directory to clone the repository: -```yml +```yaml variables: GIT_CLONE_PATH: $CI_BUILDS_DIR/project-name @@ -3898,12 +3547,12 @@ The `GIT_CLONE_PATH` has to always be within `$CI_BUILDS_DIR`. The directory set is dependent on executor and configuration of [runners.builds_dir](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) setting. -### Handling concurrency +#### Handling concurrency An executor using a concurrency greater than `1` might lead to failures because multiple jobs might be working on the same directory if the `builds_dir` is shared between jobs. -GitLab Runner does not try to prevent this situation. It is up to the administrator +GitLab Runner does not try to prevent this situation. It's up to the administrator and developers to comply with the requirements of Runner configuration. To avoid this scenario, you can use a unique path within `$CI_BUILDS_DIR`, because Runner @@ -3915,7 +3564,7 @@ exposes two additional variables that provide a unique `ID` of concurrency: The most stable configuration that should work well in any scenario and on any executor is to use `$CI_CONCURRENT_ID` in the `GIT_CLONE_PATH`. For example: -```yml +```yaml variables: GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/project-name @@ -3927,7 +3576,7 @@ test: The `$CI_CONCURRENT_PROJECT_ID` should be used in conjunction with `$CI_PROJECT_PATH` as the `$CI_PROJECT_PATH` provides a path of a repository. That is, `group/subgroup/project`. For example: -```yml +```yaml variables: GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/$CI_PROJECT_PATH @@ -3936,15 +3585,15 @@ test: - pwd ``` -### Nested paths +#### Nested paths The value of `GIT_CLONE_PATH` is expanded once and nesting variables -within it is not supported. +within is not supported. For example, you define both the variables below in your `.gitlab-ci.yml` file: -```yml +```yaml variables: GOPATH: $CI_BUILDS_DIR/go GIT_CLONE_PATH: $GOPATH/src/namespace/project @@ -3962,39 +3611,13 @@ of `.gitlab-ci.yml`. Read more about the various [YAML features](https://learnxinyminutes.com/docs/yaml/). -### Hidden keys (jobs) - -> Introduced in GitLab 8.6 and GitLab Runner v1.1.1. - -If you want to temporarily 'disable' a job, rather than commenting out all the -lines where the job is defined: - -```yaml -#hidden_job: -# script: -# - run test -``` - -you can instead start its name with a dot (`.`) and it will not be processed by -GitLab CI/CD. In the following example, `.hidden_job` will be ignored: - -```yaml -.hidden_job: - script: - - run test -``` - -Use this feature to ignore jobs, or use the -[special YAML features](#special-yaml-features) and transform the hidden keys -into templates. - ### Anchors > Introduced in GitLab 8.6 and GitLab Runner v1.1.1. YAML has a handy feature called 'anchors', which lets you easily duplicate content across your document. Anchors can be used to duplicate/inherit -properties, and is a perfect example to be used with [hidden keys](#hidden-keys-jobs) +properties, and is a perfect example to be used with [hidden jobs](#hide-jobs) to provide templates for your jobs. The following example uses anchors and map merging. It will create two jobs, @@ -4108,7 +3731,7 @@ test:mysql: - ruby ``` -You can see that the hidden keys are conveniently used as templates. +You can see that the hidden jobs are conveniently used as templates. NOTE: **Note:** You can't use YAML anchors across multiple files when leveraging the [`include`](#include) @@ -4183,14 +3806,39 @@ job_no_git_strategy: script: echo $SAMPLE_VARIABLE ``` -## Triggers +### Hide jobs -Triggers can be used to force a rebuild of a specific branch, tag or commit, -with an API call when a pipeline gets created using a trigger token. +> Introduced in GitLab 8.6 and GitLab Runner v1.1.1. -Not to be confused with [`trigger`](#trigger). +If you want to temporarily 'disable' a job, rather than commenting out all the +lines where the job is defined: -[Read more in the triggers documentation.](../triggers/README.md) +```yaml +#hidden_job: +# script: +# - run test +``` + +you can instead start its name with a dot (`.`) and it won't be processed by +GitLab CI/CD. In the following example, `.hidden_job` will be ignored: + +```yaml +.hidden_job: + script: + - run test +``` + +Use this feature to ignore jobs, or use the +[special YAML features](#special-yaml-features) and transform the hidden jobs +into templates. + +## Skip Pipeline + +If your commit message contains `[ci skip]` or `[skip ci]`, using any +capitalization, the commit will be created but the pipeline will be skipped. + +Alternatively, one can pass the `ci.skip` [Git push option](../../user/project/push_options.md#push-options-for-gitlab-cicd) +if using Git 2.10 or newer. ## Processing Git pushes @@ -4201,13 +3849,42 @@ This limitation does not affect any of the updated Merge Request pipelines, all updated Merge Requests will have a pipeline created when using [pipelines for merge requests](../merge_request_pipelines/index.md). -## Skipping jobs +## Deprecated parameters -If your commit message contains `[ci skip]` or `[skip ci]`, using any -capitalization, the commit will be created but the pipeline will be skipped. +The following parameters are deprecated. -Alternatively, one can pass the `ci.skip` [Git push option](../../user/project/push_options.md#push-options-for-gitlab-cicd) -if using Git 2.10 or newer. +### Globally-defined `types` + +CAUTION: **Deprecated:** +`types` is deprecated, and could be removed in a future release. +Use [`stages`](#stages) instead. + +### Job-defined `type` + +CAUTION: **Deprecated:** +`type` is deprecated, and could be removed in one of the future releases. +Use [`stage`](#stage) instead. + +### Globally-defined `image`, `services`, `cache`, `before_script`, `after_script` + +Defining `image`, `services`, `cache`, `before_script`, and +`after_script` globally is deprecated. Support could be removed +from a future release. + +Use [`default:`](#global-defaults) instead. For example: + +```yaml +default: + image: ruby:2.5 + services: + - docker:dind + cache: + paths: [vendor/] + before_script: + - bundle install --path vendor/ + after_script: + - rm -rf tmp/ +``` <!-- ## Troubleshooting diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md new file mode 100644 index 00000000000..a7b626bdd7c --- /dev/null +++ b/doc/ci/yaml/includes.md @@ -0,0 +1,213 @@ +# GitLab CI/CD YAML includes + +In addition to the [`includes` examples](README.md#include) listed in the +[GitLab CI YAML reference](README.md), this page lists more variations of `include` +usage. + +## Single string or array of multiple values + +You can include your extra YAML file(s) either as a single string or +an array of multiple values. The following examples are all valid. + +Single string with the `include:local` method implied: + +```yaml +include: '/templates/.after-script-template.yml' +``` + +Array with `include` method implied: + +```yaml +include: + - 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' + - '/templates/.after-script-template.yml' +``` + +Single string with `include` method specified explicitly: + +```yaml +include: + remote: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' +``` + +Array with `include:remote` being the single item: + +```yaml +include: + - remote: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' +``` + +Array with multiple `include` methods specified explicitly: + +```yaml +include: + - remote: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' + - local: '/templates/.after-script-template.yml' + - template: Auto-DevOps.gitlab-ci.yml +``` + +Array mixed syntax: + +```yaml +include: + - 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' + - '/templates/.after-script-template.yml' + - template: Auto-DevOps.gitlab-ci.yml + - project: 'my-group/my-project' + ref: master + file: '/templates/.gitlab-ci-template.yml' +``` + +## Re-using a `before_script` template + +In the following example, the content of `.before-script-template.yml` will be +automatically fetched and evaluated along with the content of `.gitlab-ci.yml`. + +Content of `https://gitlab.com/awesome-project/raw/master/.before-script-template.yml`: + +```yaml +before_script: + - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs + - gem install bundler --no-document + - bundle install --jobs $(nproc) "${FLAGS[@]}" +``` + +Content of `.gitlab-ci.yml`: + +```yaml +include: 'https://gitlab.com/awesome-project/raw/master/.before-script-template.yml' + +rspec: + script: + - bundle exec rspec +``` + +## Overriding external template values + +The following example shows specific YAML-defined variables and details of the +`production` job from an include file being customized in `.gitlab-ci.yml`. + +Content of `https://company.com/autodevops-template.yml`: + +```yaml +variables: + POSTGRES_USER: user + POSTGRES_PASSWORD: testing_password + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + +production: + stage: production + script: + - install_dependencies + - deploy + environment: + name: production + url: https://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + only: + - master +``` + +Content of `.gitlab-ci.yml`: + +```yaml +include: 'https://company.com/autodevops-template.yml' + +image: alpine:latest + +variables: + POSTGRES_USER: root + POSTGRES_PASSWORD: secure_password + +stages: + - build + - test + - production + +production: + environment: + url: https://domain.com +``` + +In this case, the variables `POSTGRES_USER` and `POSTGRES_PASSWORD` along +with the environment URL of the `production` job defined in +`autodevops-template.yml` have been overridden by new values defined in +`.gitlab-ci.yml`. + +The merging lets you extend and override dictionary mappings, but +you cannot add or modify items to an included array. For example, to add +an additional item to the production job script, you must repeat the +existing script items: + +Content of `https://company.com/autodevops-template.yml`: + +```yaml +production: + stage: production + script: + - install_dependencies + - deploy +``` + +Content of `.gitlab-ci.yml`: + +```yaml +include: 'https://company.com/autodevops-template.yml' + +stages: + - production + +production: + script: + - install_dependencies + - deploy + - notify_owner +``` + +In this case, if `install_dependencies` and `deploy` were not repeated in +`.gitlab-ci.yml`, they would not be part of the script for the `production` +job in the combined CI configuration. + +## Using nested includes + +The examples below show how includes can be nested from different sources +using a combination of different methods. + +In this example, `.gitlab-ci.yml` includes local the file `/.gitlab-ci/another-config.yml`: + +```yaml +include: + - local: /.gitlab-ci/another-config.yml +``` + +The `/.gitlab-ci/another-config.yml` includes a template and the `/templates/docker-workflow.yml` file +from another project: + +```yaml +include: + - template: Bash.gitlab-ci.yml + - project: group/my-project + file: /templates/docker-workflow.yml +``` + +The `/templates/docker-workflow.yml` present in `group/my-project` includes two local files +of the `group/my-project`: + +```yaml +include: + - local: /templates/docker-build.yml + - local: /templates/docker-testing.yml +``` + +Our `/templates/docker-build.yml` present in `group/my-project` adds a `docker-build` job: + +```yaml +docker-build: + script: docker build -t my-image . +``` + +Our second `/templates/docker-test.yml` present in `group/my-project` adds a `docker-test` job: + +```yaml +docker-test: + script: docker run my-image /run/tests.sh +``` diff --git a/doc/development/README.md b/doc/development/README.md index b041e48f0c2..22cc21e12b9 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -9,7 +9,7 @@ description: 'Learn how to contribute to GitLab.' - Set up GitLab's development environment with [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/README.md) - [GitLab contributing guide](contributing/index.md) - - [Issues workflow](contributing/issue_workflow.md). For information on: + - [Issues workflow](contributing/issue_workflow.md) for more information on: - Issue tracker guidelines. - Triaging. - Labels. @@ -17,7 +17,7 @@ description: 'Learn how to contribute to GitLab.' - Issue weight. - Regression issues. - Technical or UX debt. - - [Merge requests workflow](contributing/merge_request_workflow.md). For + - [Merge requests workflow](contributing/merge_request_workflow.md) for more information on: - Merge request guidelines. - Contribution acceptance criteria. @@ -57,10 +57,8 @@ Complementary reads: - [GitLab utilities](utilities.md) - [Issuable-like Rails models](issuable-like-models.md) - [Logging](logging.md) -- [API styleguide](api_styleguide.md) Use this styleguide if you are - contributing to the API -- [GraphQL API styleguide](api_graphql_styleguide.md) Use this - styleguide if you are contributing to the [GraphQL API](../api/graphql/index.md) +- [API style guide](api_styleguide.md) for contributing to the API +- [GraphQL API style guide](api_graphql_styleguide.md) for contributing to the [GraphQL API](../api/graphql/index.md) - [Sidekiq guidelines](sidekiq_style_guide.md) for working with Sidekiq workers - [Working with Gitaly](gitaly.md) - [Manage feature flags](feature_flags/index.md) @@ -100,6 +98,7 @@ Complementary reads: - [Rails initializers](rails_initializers.md) - [Code comments](code_comments.md) - [Renaming features](renaming_features.md) +- [Windows Development on GCP](windows.md) ## Performance guides @@ -117,7 +116,7 @@ Complementary reads: ### Tooling - [Understanding EXPLAIN plans](understanding_explain_plans.md) -- [explain.depesz.com](https://explain.depesz.com/) for visualising the output +- [explain.depesz.com](https://explain.depesz.com/) for visualizing the output of `EXPLAIN` - [pgFormatter](http://sqlformat.darold.net/) a PostgreSQL SQL syntax beautifier @@ -178,7 +177,7 @@ Complementary reads: ## Documentation guides - [Writing documentation](documentation/index.md) -- [Documentation styleguide](documentation/styleguide.md) +- [Documentation style guide](documentation/styleguide.md) - [Markdown](../user/markdown.md) ## Internationalization (i18n) guides @@ -189,11 +188,11 @@ Complementary reads: ## Telemetry guides -- [Introduction](../telemetry/index.md) -- [Frontend tracking guide](../telemetry/frontend.md) -- [Backend tracking guide](../telemetry/backend.md) +- [Telemetry guide](telemetry/index.md) +- [Usage Ping guide](telemetry/usage_ping.md) +- [Snowplow guide](telemetry/snowplow.md) -## Experiment Guide +## Experiment guide - [Introduction](experiment_guide/index.md) @@ -221,9 +220,10 @@ Complementary reads: - [Defining relations between files using projections](projections.md) - [Reference processing](./reference_processing.md) +- [Compatibility with multiple versions of the application running at the same time](multi_version_compatibility.md) ## Other GitLab Development Kit (GDK) guides - [Run full Auto DevOps cycle in a GDK instance](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/auto_devops.md) -- [Using GitLab Runner with GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/runner.md) -- [Using the Web IDE terminal with GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/web_ide_terminal_gdk_setup.md) +- [Using GitLab Runner with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/runner.md) +- [Using the Web IDE terminal with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/web_ide_terminal_gdk_setup.md) diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 036eddd7c37..6d3c0cf0eac 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -1,6 +1,6 @@ -# GraphQL API +# GraphQL API style guide -This document outlines the styleguide for GitLab's [GraphQL API](../api/graphql/index.md). +This document outlines the style guide for GitLab's [GraphQL API](../api/graphql/index.md). ## How GitLab implements GraphQL @@ -12,7 +12,7 @@ which is exposed as an API endpoint at `/api/graphql`. ## Deep Dive -In March 2019, Nick Thomas hosted a [Deep Dive](https://gitlab.com/gitlab-org/create-stage/issues/1) +In March 2019, Nick Thomas hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on GitLab's [GraphQL API](../api/graphql/index.md) to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=-9L_1MWrjkg), and the slides on @@ -102,7 +102,7 @@ be `id` fields. Further reading: - [GraphQL Best Practices Guide](https://graphql.org/learn/best-practices/#nullability) -- [Using nullability in GraphQL](https://blog.apollographql.com/using-nullability-in-graphql-2254f84c4ed7) +- [Using nullability in GraphQL](https://www.apollographql.com/blog/using-nullability-in-graphql-2254f84c4ed7) ### Exposing Global IDs @@ -110,7 +110,7 @@ When exposing an `ID` field on a type, we will by default try to expose a global ID by calling `to_global_id` on the resource being rendered. -To override this behaviour, you can implement an `id` method on the +To override this behavior, you can implement an `id` method on the type for which you are exposing an ID. Please make sure that when exposing a `GraphQL::ID_TYPE` using a custom method that it is globally unique. @@ -365,7 +365,7 @@ field :token, GraphQL::STRING_TYPE, null: true, The original `description:` of the field should be maintained, and should _not_ be updated to mention the deprecation. -### Deprecation reason styleguide +### Deprecation reason style guide Where the reason for deprecation is due to the field being replaced with another field, the `reason` must be: @@ -446,7 +446,7 @@ Descriptions of fields and arguments are viewable to users through: - The [GraphiQL explorer](#graphiql). - The [static GraphQL API reference](../api/graphql/#reference). -### Description styleguide +### Description style guide To ensure consistency, the following should be followed whenever adding or updating descriptions: @@ -598,7 +598,7 @@ ID. Otherwise use a [globally unique ID](#exposing-global-ids). We already have a `FullPathLoader` that can be included in other resolvers to quickly find Projects and Namespaces which will have a -lot of dependant objects. +lot of dependent objects. To limit the amount of queries performed, we can use `BatchLoader`. diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index 78e35023766..e9a41e7ec58 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -1,6 +1,6 @@ -# API styleguide +# API style guide -This styleguide recommends best practices for API development. +This style guide recommends best practices for API development. ## Instance variables @@ -9,7 +9,7 @@ to access them as we do in Rails views), local variables are fine. ## Entities -Always use an [Entity] to present the endpoint's payload. +Always use an [Entity](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/api/entities) to present the endpoint's payload. ## Documentation @@ -29,7 +29,7 @@ for a good example): - If the endpoint is deprecated, and if so, when will it be removed - `params` for the method parameters. This acts as description, - [validation, and coercion of the parameters] + [validation, and coercion of the parameters](https://github.com/ruby-grape/grape#parameter-validation-and-coercion) A good example is as follows: @@ -100,13 +100,13 @@ Model.create(foo: params[:foo]) ## Using HTTP status helpers -For non-200 HTTP responses, use the provided helpers in `lib/api/helpers.rb` to ensure correct behaviour (`not_found!`, `no_content!` etc.). These will `throw` inside Grape and abort the execution of your endpoint. +For non-200 HTTP responses, use the provided helpers in `lib/api/helpers.rb` to ensure correct behavior (`not_found!`, `no_content!` etc.). These will `throw` inside Grape and abort the execution of your endpoint. For `DELETE` requests, you should also generally use the `destroy_conditionally!` helper which by default returns a `204 No Content` response on success, or a `412 Precondition Failed` response if the given `If-Unmodified-Since` header is out of range. This helper calls `#destroy` on the passed resource, but you can also implement a custom deletion method by passing a block. ## Using API path helpers in GitLab Rails codebase -Because we support [installing GitLab under a relative URL], one must take this +Because we support [installing GitLab under a relative URL](../install/relative_url.md), one must take this into account when using API path helpers generated by Grape. Any such API path helper usage must be in wrapped into the `expose_path` helper call. @@ -121,10 +121,6 @@ For instance: The [internal API](./internal_api.md) is documented for internal use. Please keep it up to date so we know what endpoints different components are making use of. -[Entity]: https://gitlab.com/gitlab-org/gitlab/blob/master/lib/api/entities -[validation, and coercion of the parameters]: https://github.com/ruby-grape/grape#parameter-validation-and-coercion -[installing GitLab under a relative URL]: https://docs.gitlab.com/ee/install/relative_url.html - ## Avoiding N+1 problems In order to avoid N+1 problems that are common when returning collections @@ -146,7 +142,7 @@ data](https://gitlab.com/gitlab-org/gitlab/blob/19f74903240e209736c7668132e6a5a7 for `Todo` _targets_ when returned in the Todos API. For more context and discussion about preloading see -[this merge request](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/25711) +[this merge request](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25711) which introduced the scope. ### Verifying with tests diff --git a/doc/development/application_limits.md b/doc/development/application_limits.md index b6e777dee15..b13e2994c52 100644 --- a/doc/development/application_limits.md +++ b/doc/development/application_limits.md @@ -11,7 +11,7 @@ coordinate with others to [document](../administration/instance_limits.md) and communicate those limits. There is a guide about [introducing application -limits](https://about.gitlab.com/handbook/product/#introducing-application-limits). +limits](https://about.gitlab.com/handbook/product/product-management/process/index.html#introducing-application-limits). ## Development @@ -90,7 +90,7 @@ project.actual_limits.exceeded?(:project_hooks, 10) #### `Limitable` concern -The [`Limitable` concern](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/concerns/limitable.rb) +The [`Limitable` concern](https://gitlab.com/gitlab-org/gitlab/blob/master/app/models/concerns/limitable.rb) can be used to validate that a model does not exceed the limits. It ensures that the count of the records for the current model does not exceed the defined limit. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index bbd5ca3c494..f52a5d30c1f 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -8,7 +8,7 @@ New versions of GitLab are released in stable branches and the master branch is For information, see the [GitLab Release Process](https://gitlab.com/gitlab-org/release/docs/-/tree/master#gitlab-release-process). -Both EE and CE require some add-on components called GitLab Shell and Gitaly. These components are available from the [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell/-/tree/master) and [Gitaly](https://gitlab.com/gitlab-org/gitaly/-/tree/master) repositories respectively. New versions are usually tags but staying on the master branch will give you the latest stable version. New releases are generally around the same time as GitLab CE releases with exception for informal security updates deemed critical. +Both EE and CE require some add-on components called GitLab Shell and Gitaly. These components are available from the [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell/-/tree/master) and [Gitaly](https://gitlab.com/gitlab-org/gitaly/-/tree/master) repositories respectively. New versions are usually tags but staying on the master branch will give you the latest stable version. New releases are generally around the same time as GitLab CE releases with the exception of informal security updates deemed critical. ## Components diff --git a/doc/development/auto_devops.md b/doc/development/auto_devops.md index dba68974dd5..7d0c020ef96 100644 --- a/doc/development/auto_devops.md +++ b/doc/development/auto_devops.md @@ -1,7 +1,11 @@ # Auto DevOps development guide This document provides a development guide for contributors to -[Auto DevOps](../topics/autodevops/index.md) +[Auto DevOps](../topics/autodevops/index.md). + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +An [Auto DevOps technical walk-through](https://youtu.be/G7RTLeToz9E) +is also available on YouTube. ## Development diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index c28008c94b8..0747224db30 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -157,7 +157,7 @@ roughly be as follows: enough times to be marked as dead. 1. Remove the old column. -This may also require a bump to the [import/export version][import-export], if +This may also require a bump to the [import/export version](../user/project/settings/import_export.md), if importing a project from a prior version of GitLab requires the data to be in the new format. @@ -283,7 +283,7 @@ end The final step runs for any un-migrated rows after all of the jobs have been processed. This is in case a Sidekiq process running the background migrations received SIGKILL, leading to the jobs being lost. (See -[more reliable Sidekiq queue][reliable-sidekiq] for more information.) +[more reliable Sidekiq queue](https://gitlab.com/gitlab-org/gitlab-foss/issues/36791) for more information.) If the application does not depend on the data being 100% migrated (for instance, the data is advisory, and not mission-critical), then this final step @@ -312,7 +312,7 @@ to migrate you database down and up, which can result in other background migrations being called. That means that using `spy` test doubles with `have_received` is encouraged, instead of using regular test doubles, because your expectations defined in a `it` block can conflict with what is being -called in RSpec hooks. See [issue #35351][issue-rspec-hooks] +called in RSpec hooks. See [issue #35351](https://gitlab.com/gitlab-org/gitlab/issues/18839) for more details. ## Best practices @@ -329,8 +329,3 @@ for more details. 1. Make sure to discuss the numbers with a database specialist, the migration may add more pressure on DB than you expect (measure on staging, or ask someone to measure on production). - -[migrations-readme]: https://gitlab.com/gitlab-org/gitlab/blob/master/spec/migrations/README.md -[issue-rspec-hooks]: https://gitlab.com/gitlab-org/gitlab/issues/18839 -[reliable-sidekiq]: https://gitlab.com/gitlab-org/gitlab-foss/issues/36791 -[import-export]: ../user/project/settings/import_export.md diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 2007c26403c..a0e9f9d3be3 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -101,20 +101,20 @@ automatically. Its simplest usage is to provide the value for `title`: -```text +```plaintext bin/changelog 'Hey DZ, I added a feature to GitLab!' ``` If you want to generate a changelog entry for GitLab EE, you will need to pass the `--ee` option: -```text +```plaintext bin/changelog --ee 'Hey DZ, I added a feature to GitLab!' ``` At this point the script would ask you to select the category of the change (mapped to the `type` field in the entry): -```text +```plaintext >> Please specify the category of your change: 1. New feature 2. Bug fix @@ -132,7 +132,7 @@ the command above on a branch called `feature/hey-dz`, it will generate a The command will output the path of the generated file and its contents: -```text +```plaintext create changelogs/unreleased/my-feature.yml --- title: Hey DZ, I added a feature to GitLab! @@ -162,7 +162,7 @@ If you use **`--amend`** and don't provide a title, it will automatically use the "subject" of the previous commit, which is the first line of the commit message: -```text +```plaintext $ git show --oneline ab88683 Added an awesome new feature to GitLab @@ -180,7 +180,7 @@ type: Use **`--force`** or **`-f`** to overwrite an existing changelog entry if it already exists. -```text +```plaintext $ bin/changelog 'Hey DZ, I added a feature to GitLab!' error changelogs/unreleased/feature-hey-dz.yml already exists! Use `--force` to overwrite. @@ -198,7 +198,7 @@ type: Use the **`--merge-request`** or **`-m`** argument to provide the `merge_request` value: -```text +```plaintext $ bin/changelog 'Hey DZ, I added a feature to GitLab!' -m 1983 create changelogs/unreleased/feature-hey-dz.yml --- @@ -213,7 +213,7 @@ type: Use the **`--dry-run`** or **`-n`** argument to prevent actually writing or committing anything: -```text +```plaintext $ bin/changelog --amend --dry-run create changelogs/unreleased/feature-hey-dz.yml --- @@ -230,7 +230,7 @@ $ ls changelogs/unreleased/ Use the **`--git-username`** or **`-u`** argument to automatically fill in the `author` value with your configured Git `user.name` value: -```text +```plaintext $ git config user.name Jane Doe @@ -247,7 +247,7 @@ type: Use the **`--type`** or **`-t`** argument to provide the `type` value: -```text +```plaintext $ bin/changelog 'Hey DZ, I added a feature to GitLab!' -t added create changelogs/unreleased/feature-hey-dz.yml --- diff --git a/doc/development/cicd/img/ci_architecture.png b/doc/development/cicd/img/ci_architecture.png Binary files differnew file mode 100644 index 00000000000..b888f2f07aa --- /dev/null +++ b/doc/development/cicd/img/ci_architecture.png diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index ed33eb01308..db3f58d1d22 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -2,6 +2,81 @@ Development guides that are specific to CI/CD are listed here. +## CI Architecture overview + +The following is a simplified diagram of the CI architecture. Some details are left out in order to focus on +the main components. + + +<!-- Editable diagram available at https://app.diagrams.net/#G1LFl-KW4fgpBPzz8VIH9rsOlAH4t0xwKj --> + +On the left side we have the events that can trigger a pipeline based on various events (trigged by a user or automation): + +- A `git push` is the most common event that triggers a pipeline. +- The [Web API](../../api/pipelines.md#create-a-new-pipeline). +- A user clicking the "Run Pipeline" button in the UI. +- When a [merge request is created or updated](../../ci/merge_request_pipelines/index.md#pipelines-for-merge-requests). +- When an MR is added to a [Merge Train](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md#merge-trains-premium). +- A [scheduled pipeline](../../ci/pipelines/schedules.md#pipeline-schedules). +- When project is [subscribed to an upstream project](../../ci/multi_project_pipelines.md#trigger-a-pipeline-when-an-upstream-project-is-rebuilt). +- When [Auto DevOps](../../topics/autodevops/index.md) is enabled. +- When GitHub integration is used with [external pull requests](../../ci/ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). +- When an upstream pipeline contains a [bridge job](../../ci/yaml/README.md#trigger) which triggers a downstream pipeline. + +Triggering any of these events will invoke the [`CreatePipelineService`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/ci/create_pipeline_service.rb) +which takes as input event data and the user triggering it, then will attempt to create a pipeline. + +The `CreatePipelineService` relies heavily on the [`YAML Processor`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/yaml_processor.rb) +component, which is responsible for taking in a YAML blob as input and returns the abstract data structure of a +pipeline (including stages and all jobs). This component also validates the structure of the YAML while +processing it, and returns any syntax or semantic errors. The `YAML Processor` component is where we define +[all the keywords](../../ci/yaml/README.md) available to structure a pipeline. + +The `CreatePipelineService` receives the abstract data structure returned by the `YAML Processor`, +which then converts it to persisted models (pipeline, stages, jobs, etc.). After that, the pipeline is ready +to be processed. Processing a pipeline means running the jobs in order of execution (stage or DAG) +until either one of the following: + +- All expected jobs have been executed. +- Failures interrupt the pipeline execution. + +The component that processes a pipeline is [`ProcessPipelineService`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/ci/process_pipeline_service.rb), +which is responsible for moving all the pipeline's jobs to a completed state. When a pipeline is created, all its +jobs are initially in `created` state. This services looks at what jobs in `created` stage are eligible +to be processed based on the pipeline structure. Then it moves them into the `pending` state, which means +they can now [be picked up by a Runner](#job-scheduling). After a job has been executed it can complete +successfully or fail. Each status transition for job within a pipeline triggers this service again, which +looks for the next jobs to be transitioned towards completion. While doing that, `ProcessPipelineService` +updates the status of jobs, stages and the overall pipeline. + +On the right side of the diagram we have a list of [Runners](../../ci/runners/README.md#configuring-gitlab-runners) +connected to the GitLab instance. These can be Shared Runners, Group Runners or Project-specific Runners. +The communication between Runners and the Rails server occurs through a set of API endpoints, grouped as +the `Runner API Gateway`. + +We can register, delete and verify Runners, which also causes read/write queries to the database. After a Runner is connected, +it keeps asking for the next job to execute. This invokes the [`RegisterJobService`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/services/ci/register_job_service.rb) +which will pick the next job and assign it to the Runner. At this point the job will transition to a +`running` state, which again triggers `ProcessPipelineService` due to the status change. +For more details read [Job scheduling](#job-scheduling)). + +While a job is being executed, the Runner sends logs back to the server as well any possible artifacts +that need to be stored. Also, a job may depend on artifacts from previous jobs in order to run. In this +case the Runner will download them using a dedicated API endpoint. + +Artifacts are stored in object storage, while metadata is kept in the database. An important example of artifacts +is reports (JUnit, SAST, DAST, etc.) which are parsed and rendered in the merge request. + +Job status transitions are not all automated. A user may run [manual jobs](../../ci/yaml/README.md#whenmanual), cancel a pipeline, retry +specific failed jobs or the entire pipeline. Anything that +causes a job to change status will trigger `ProcessPipelineService`, as it's responsible for +tracking the status of the entire pipeline. + +A special type of job is the [bridge job](../../ci/yaml/README.md#trigger) which is executed server-side +when transitioning to the `pending` state. This job is responsible for creating a downstream pipeline, such as +a multi-project or child pipeline. The workflow loop starts again +from the `CreatePipelineService` every time a downstream pipeline is triggered. + ## Job scheduling When a Pipeline is created all its jobs are created at once for all stages, with an initial state of `created`. This makes it possible to visualize the full content of a pipeline. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 5220f29bd60..a5ad7dc0f46 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -64,7 +64,7 @@ It picks reviewers and maintainers from the list at the page, with these behaviors: 1. It will not pick people whose [GitLab status](../user/profile/index.md#current-status) - contains the string 'OOO'. + contains the string 'OOO', or the emoji is `:palm_tree:` or `:beach:`. 1. [Trainee maintainers](https://about.gitlab.com/handbook/engineering/workflow/code-review/#trainee-maintainer) are three times as likely to be picked as other reviewers. 1. It always picks the same reviewers and maintainers for the same @@ -76,26 +76,36 @@ page, with these behaviors: As described in the section on the responsibility of the maintainer below, you are recommended to get your merge request approved and merged by maintainer(s) - with [domain expertise](#domain-experts). +with [domain expertise](#domain-experts). -1. If your merge request includes backend changes [^1], it must be +1. If your merge request includes backend changes (*1*), it must be **approved by a [backend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_backend)**. -1. If your merge request includes database migrations or changes to expensive queries [^2], it must be +1. If your merge request includes database migrations or changes to expensive queries (*2*), it must be **approved by a [database maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_database)**. Read the [database review guidelines](database_review.md) for more details. -1. If your merge request includes frontend changes [^1], it must be +1. If your merge request includes frontend changes (*1*), it must be **approved by a [frontend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_frontend)**. -1. If your merge request includes UX changes [^1], it must be +1. If your merge request includes UX changes (*1*), it must be **approved by a [UX team member](https://about.gitlab.com/company/team/)**. -1. If your merge request includes adding a new JavaScript library [^1], it must be +1. If your merge request includes adding a new JavaScript library (*1*), it must be **approved by a [frontend lead](https://about.gitlab.com/company/team/)**. -1. If your merge request includes adding a new UI/UX paradigm [^1], it must be +1. If your merge request includes adding a new UI/UX paradigm (*1*), it must be **approved by a [UX lead](https://about.gitlab.com/company/team/)**. 1. If your merge request includes a new dependency or a filesystem change, it must be **approved by a [Distribution team member](https://about.gitlab.com/company/team/)**. See how to work with the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/#how-to-work-with-distribution) for more details. 1. If your merge request includes documentation changes, it must be **approved by a [Technical writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers)**, based on the appropriate [product category](https://about.gitlab.com/handbook/product/categories/). +1. If your merge request includes Quality and non-Quality-related changes (*3*), it must be **approved + by a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors)**. +1. If your merge request includes _only_ Quality-related changes (*3*), it must be **approved + by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)**. + +- (*1*): Please note that specs other than JavaScript specs are considered backend code. +- (*2*): We encourage you to seek guidance from a database maintainer if your merge + request is potentially introducing expensive queries. It is most efficient to comment + on the line of code in question with the SQL queries so they can give their advice. +- (*3*): Quality-related changes include all files within the `qa` directory. #### Security requirements @@ -314,7 +324,25 @@ Before taking the decision to merge: - Set the milestone. - Consider warnings and errors from danger bot, code quality, and other reports. Unless a strong case can be made for the violation, these should be resolved - before merge. A comment must to be posted if the MR is merged with any failed job. + before merging. A comment must to be posted if the MR is merged with any failed job. +- If the MR contains both Quality and non-Quality-related changes, the MR should be merged by the relevant maintainer for user-facing changes (backend, frontend, or database) after the Quality related changes are approved by a Software Engineer in Test. + +If a merge request is fundamentally ready, but needs only trivial fixes (such as +typos), consider demonstrating a [bias for +action](https://about.gitlab.com/handbook/values/#bias-for-action) by making +those changes directly without going back to the author. You can do this by +using the [suggest changes](../user/discussions/index.md#suggest-changes) feature to apply +your own suggestions to the merge request. Note that: + +- If the changes are not straightforward, please prefer assigning the merge request back + to the author. +- **Before applying suggestions**, edit the merge request to make sure + [squash and + merge](../user/project/merge_requests/squash_and_merge.md#squash-and-merge) + is enabled, otherwise, the pipeline's Danger job will fail. + - If a merge request does not have squash and merge enabled, and it + has more than one commit, then see the note below about rewriting + commit history. When ready to merge: @@ -463,6 +491,21 @@ When a merge request author has been blocked for longer than the `Review-response` SLO, they are free to remind the reviewer through Slack or assign another reviewer. +### Customer critical merge requests + +A merge request may benefit from being considered a customer critical priority because there is a significant benefit to the business in doing so. + +Properties of customer critical merge requests: + +- The [Senior Director of Development](https://about.gitlab.com/job-families/engineering/engineering-management/#senior-director-engineering) ([@clefelhocz1](https://gitlab.com/clefelhocz1)) is the DRI for deciding if a merge request will be customer critical. +- The DRI will assign the `customer-critical-merge-request` label to the merge request. +- It is required that the reviewer(s) and maintainer(s) involved with a customer critical merge request are engaged as soon as this decision is made. +- It is required to prioritize work for those involved on a customer critical merge request so that they have the time available necessary to focus on it. +- It is required to adhere to GitLab [values](https://about.gitlab.com/handbook/values.md) and processes when working on customer critical merge requests, taking particular note of family and friends first/work second, definition of done, iteration, and release when it's ready. +- Customer critical merge requests are required to not reduce security, introduce data-loss risk, reduce availability, nor break existing functionality per the process for [prioritizing technical decisions](https://about.gitlab.com/handbook/engineering/#prioritizing-technical-decisions.md). +- On customer critical requests, it is _recommended_ that those involved _consider_ coordinating synchronously (Zoom, Slack) in addition to asynchronously (merge requests comments) if they believe this will reduce elapsed time to merge even though this _may_ sacrifice [efficiency](https://about.gitlab.com/company/culture/all-remote/asynchronous/#evaluating-efficiency.md). +- After a customer critical merge request is merged, a retrospective must be completed with the intention of reducing the frequency of future customer critical merge requests. + ## Examples How code reviews are conducted can surprise new contributors. Here are some examples of code reviews that should help to orient you as to what to expect. @@ -495,6 +538,3 @@ Largely based on the [thoughtbot code review guide](https://github.com/thoughtbo --- [Return to Development documentation](README.md) - -[^1]: Please note that specs other than JavaScript specs are considered backend code. -[^2]: We encourage you to seek guidance from a database maintainer if your merge request is potentially introducing expensive queries. It is most efficient to comment on the line of code in question with the SQL queries so they can give their advice. diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index 8270e52e0ab..aebff633f58 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -3,35 +3,40 @@ Thank you for your interest in contributing to GitLab. This guide details how to contribute to GitLab in a way that is easy for everyone. -For a first-time step-by-step guide to the contribution process, please see -["Contributing to GitLab"](https://about.gitlab.com/community/contribute/). +For a first-time step-by-step guide to the contribution process, see our +[Contributing to GitLab](https://about.gitlab.com/community/contribute/) page. -Looking for something to work on? Look for issues with the label [`Accepting merge requests`](#i-want-to-contribute). +Looking for something to work on? Look for issues with the label +[`~Accepting merge requests`](#how-to-contribute). -GitLab comes in two flavors, GitLab Community Edition (CE) our free and open -source edition, and GitLab Enterprise Edition (EE) which is our commercial -edition. Throughout this guide you will see references to CE and EE for -abbreviation. +GitLab comes in two flavors: -To get an overview of GitLab community membership including those that would be reviewing or merging your contributions, please visit [the community roles page](community_roles.md). +- GitLab Community Edition (CE), our free and open source edition. +- GitLab Enterprise Edition (EE), which is our commercial edition. + +Throughout this guide you will see references to CE and EE for abbreviation. + +To get an overview of GitLab community membership, including those that would review or merge +your contributions, visit [the community roles page](community_roles.md). If you want to know how the GitLab [core team](https://about.gitlab.com/community/core-team/) -operates please see [the GitLab contributing process](https://gitlab.com/gitlab-org/gitlab/blob/master/PROCESS.md). +operates, see [the GitLab contributing process](https://gitlab.com/gitlab-org/gitlab/blob/master/PROCESS.md). -[GitLab Inc engineers should refer to the engineering workflow document](https://about.gitlab.com/handbook/engineering/workflow/) +GitLab Inc engineers should refer to the [engineering workflow document](https://about.gitlab.com/handbook/engineering/workflow/). ## Security vulnerability disclosure -Please report suspected security vulnerabilities in private to +Report suspected security vulnerabilities in private to `support@gitlab.com`, also see the [disclosure section on the GitLab.com website](https://about.gitlab.com/security/disclosure/). -Please do **NOT** create publicly viewable issues for suspected security -vulnerabilities. + +DANGER: **Danger:** +Do **NOT** create publicly viewable issues for suspected security vulnerabilities. ## Code of conduct We want to create a welcoming environment for everyone who is interested in contributing. -Please visit our [Code of Conduct page](https://about.gitlab.com/community/contribute/code-of-conduct/) to learn more about our commitment to an open and welcoming environment. +Visit our [Code of Conduct page](https://about.gitlab.com/community/contribute/code-of-conduct/) to learn more about our commitment to an open and welcoming environment. ## Closing policy for issues and merge requests @@ -40,39 +45,56 @@ and merge requests is limited. Out of respect for our volunteers, issues and merge requests not in line with the guidelines listed in this document may be closed without notice. -Please treat our volunteers with courtesy and respect, it will go a long way +Treat our volunteers with courtesy and respect, it will go a long way towards getting your issue resolved. Issues and merge requests should be in English and contain appropriate language for audiences of all ages. -If a contributor is no longer actively working on a submitted merge request -we can decide that the merge request will be finished by one of our -[Merge request coaches](https://about.gitlab.com/company/team/) or close the merge request. We make this decision -based on how important the change is for our product vision. If a merge request -coach is going to finish the merge request we assign the -~"coach will finish" label. When a team member picks up a community contribution, +If a contributor is no longer actively working on a submitted merge request, +we can: + +- Decide that the merge request will be finished by one of our + [Merge request coaches](https://about.gitlab.com/company/team/). +- Close the merge request. + +We make this decision based on how important the change is for our product vision. If a merge +request coach is going to finish the merge request, we assign the +`~coach will finish` label. + +When a team member picks up a community contribution, we credit the original author by adding a changelog entry crediting the author and optionally include the original author on at least one of the commits within the MR. ## Helping others -Please help other GitLab users when you can. -The methods people will use to seek help can be found on the [getting help page](https://about.gitlab.com/get-help/). +Help other GitLab users when you can. +The methods people use to seek help can be found on the [getting help page](https://about.gitlab.com/get-help/). -Sign up for the mailing list, answer GitLab questions on StackOverflow or -respond in the IRC channel. +Sign up for the mailing list, answer GitLab questions on StackOverflow or respond in the IRC channel. -## I want to contribute +## How to contribute If you want to contribute to GitLab, -[issues with the `Accepting merge requests` label](issue_workflow.md#label-for-community-contributors) +[issues with the `~Accepting merge requests` label](issue_workflow.md#label-for-community-contributors) are a great place to start. + If you have any questions or need help visit [Getting Help](https://about.gitlab.com/get-help/) to -learn how to communicate with GitLab. If you're looking for a Gitter or Slack channel -please consider we favor -[asynchronous communication](https://about.gitlab.com/handbook/communication/#internal-communication) over real time communication. Thanks for your contribution! +learn how to communicate with GitLab. We have a [Gitter channel for contributors](https://gitter.im/gitlab/contributors), +however we favor +[asynchronous communication](https://about.gitlab.com/handbook/communication/#internal-communication) over real time communication. + +Thanks for your contribution! + +### GitLab Development Kit + +The GitLab Development Kit (GDK) helps contributors run a local GitLab instance with all the +required dependencies. It can be used to test changes to GitLab and related projects before raising +a Merge Request. + +For more information, see the [`gitlab-development-kit`](https://gitlab.com/gitlab-org/gitlab-development-kit) +project. ## Contribution Flow @@ -92,7 +114,7 @@ When submitting code to GitLab, you may feel that your contribution requires the When your code contains more than 500 changes, any major breaking changes, or an external library, `@mention` a maintainer in the merge request. If you are not sure who to mention, the reviewer will add one early in the merge request process. -## Issues workflow +### Issues workflow This [documentation](issue_workflow.md) outlines the current issue workflow: @@ -105,7 +127,7 @@ This [documentation](issue_workflow.md) outlines the current issue workflow: - [Technical and UX debt](issue_workflow.md#technical-and-ux-debt) - [Technical debt in follow-up issues](issue_workflow.md#technical-debt-in-follow-up-issues) -## Merge requests workflow +### Merge requests workflow This [documentation](merge_request_workflow.md) outlines the current merge request process. @@ -120,13 +142,15 @@ This [documentation](style_guides.md) outlines the current style guidelines. ## Implement design & UI elements -This [design documentation](design.md) outlines the current process for implementing -design & UI elements. +This [design documentation](design.md) outlines the current process for implementing design and UI +elements. -## Getting an Enterprise Edition License +## Contribute documentation -If you need a license for contributing to an EE-feature, please [follow these instructions](https://about.gitlab.com/handbook/marketing/community-relations/code-contributor-program/#for-contributors-to-the-gitlab-enterprise-edition-ee). +For information on how to contribute documentation, see GitLab +[documentation guidelines](../documentation/index.md). ---- +## Getting an Enterprise Edition License -[Return to Development documentation](../README.md) +If you need a license for contributing to an EE-feature, see +[relevant information](https://about.gitlab.com/handbook/marketing/community-relations/code-contributor-program/#for-contributors-to-the-gitlab-enterprise-edition-ee). diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index 13ff35ed65c..be416bf636e 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -36,7 +36,7 @@ project. To allow for asynchronous issue handling, we use [milestones](https://gitlab.com/groups/gitlab-org/-/milestones) and [labels](https://gitlab.com/gitlab-org/gitlab/-/labels). Leads and product managers handle most of the -scheduling into milestones. Labelling is a task for everyone. +scheduling into milestones. Labeling is a task for everyone. Most issues will have labels for at least one of the following: @@ -156,9 +156,9 @@ As a team needs some way to collect the work their members are planning to be as Normally there is a 1:1 relationship between Stage labels and Group labels. In the spirit of "Everyone can contribute", any issue can be picked up by any group, depending on current priorities. When picking up an issue belonging to a different -group, it should be relabelled. For example, if an issue labelled `~"devops::create"` +group, it should be relabeled. For example, if an issue labeled `~"devops::create"` and `~"group::knowledge"` is picked up by someone in the Access group of the Plan stage, -the issue should be relabelled as `~"group::access"` while keeping the original +the issue should be relabeled as `~"group::access"` while keeping the original `~"devops::create"` unchanged. We also use stage and group labels to help quantify our [throughput](https://about.gitlab.com/handbook/engineering/management/throughput/). @@ -264,7 +264,7 @@ release. There are three levels of Release Scoping labels: milestone. If these issues are not done in the current release, they will strongly be considered for the next release. - ~"Next Patch Release": Issues to put in the next patch release. Work on these - first, and add the "Pick Into X" label to the merge request, along with the + first, and add the `~"Pick into X.Y"` label to the merge request, along with the appropriate milestone. Each issue scheduled for the current milestone should be labeled ~Deliverable @@ -347,7 +347,7 @@ there is the ~"stewardship" label. This label is to be used for issues in which the stewardship of GitLab is a topic of discussion. For instance if GitLab Inc. is planning to add -features from GitLab EE to GitLab CE, related issues would be labelled with +features from GitLab EE to GitLab CE, related issues would be labeled with ~"stewardship". A recent example of this was the issue for @@ -461,7 +461,7 @@ fixing in the same MR, but not worth creating a follow-up issue for. Renaming a method that is used in many places to make its intent slightly clearer may be worth fixing, but it should not happen in the same MR, and is generally not worth the overhead of having an issue of its own. These issues would invariably -be labelled `~P4 ~S4` if we were to create them. +be labeled `~P4 ~S4` if we were to create them. More severe technical debt can have implications for development velocity. If it isn't addressed in a timely manner, the codebase becomes needlessly difficult diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 0ac08ec2eae..a70fadfe8a9 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -17,10 +17,10 @@ wireframes of the proposed feature if it will also change the UI. Merge requests should be submitted to the appropriate project at GitLab.com, for example [GitLab](https://gitlab.com/gitlab-org/gitlab/-/merge_requests), [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests), -[GitLab Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests), etc. +[Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests), etc. If you are new to GitLab development (or web development in general), see the -[I want to contribute!](index.md#i-want-to-contribute) section to get started with +[how to contribute](index.md#how-to-contribute) section to get started with some potentially easy issues. To start developing GitLab, download the [GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit) @@ -138,7 +138,7 @@ For more information see [How to Write a Git Commit Message](https://chris.beams Example commit message template that can be used on your machine that embodies the above (guide for [how to apply template](https://codeinthehole.com/tips/a-useful-template-for-commit-messages/)): -```text +```plaintext # (If applied, this commit will...) <subject> (Max 50 char) # |<---- Using a Maximum Of 50 Characters ---->| @@ -244,7 +244,7 @@ request: 1. The [CI environment preparation](https://gitlab.com/gitlab-org/gitlab/blob/master/scripts/prepare_build.sh). 1. The [Omnibus package creator](https://gitlab.com/gitlab-org/omnibus-gitlab). -### Incremental improvements +## Incremental improvements We allow engineering time to fix small problems (with or without an issue) that are incremental improvements, such as: diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index 9a0b654f47f..9e4870eadc4 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -53,7 +53,7 @@ should enable all RuboCop rules to avoid style-related discussions/nitpicking/back-and-forth in reviews. Additionally, we have a dedicated -[newlines styleguide](../newlines_styleguide.md), as well as dedicated +[newlines style guide](../newlines_styleguide.md), as well as dedicated [test-specific style guides and best practices](../testing_guide/index.md). ## Database migrations diff --git a/doc/development/creating_enums.md b/doc/development/creating_enums.md index 79ed465b121..e2ebad538d9 100644 --- a/doc/development/creating_enums.md +++ b/doc/development/creating_enums.md @@ -13,3 +13,92 @@ def change add_column :ci_job_artifacts, :file_format, :integer, limit: 2 end ``` + +## All of the key/value pairs should be defined in FOSS + +**Summary:** All enums needs to be defined in FOSS, if a model is also part of the FOSS. + +```ruby +class Model < ApplicationRecord + enum platform: { + aws: 0, + gcp: 1 # EE-only + } +end +``` + +When you add a new key/value pair to a `enum` and if it's EE-specific, you might be +tempted to organize the `enum` as the following: + +```ruby +# Define `failure_reason` enum in `Pipeline` model: +class Pipeline < ApplicationRecord + enum failure_reason: ::PipelineEnums.failure_reasons +end +``` + +```ruby +# Define key/value pairs that used in FOSS and EE: +module PipelineEnums + def self.failure_reasons + { unknown_failure: 0, config_error: 1 } + end +end + +PipelineEnums.prepend_if_ee('EE::PipelineEnums') +``` + +```ruby +# Define key/value pairs that used in EE only: +module EE + module PipelineEnums + override :failure_reasons + def failure_reasons + super.merge(activity_limit_exceeded: 2) + end + end +end +``` + +This works as-is, however, it has a couple of downside that: + +- Someone could define a key/value pair in EE that is **conflicted** with a value defined in FOSS. + e.g. Define `activity_limit_exceeded: 1` in `EE::PipelineEnums`. +- When it happens, the feature works totally different. + e.g. We cannot figure out `failure_reason` is either `config_error` or `activity_limit_exceeded`. +- When it happens, we have to ship a database migration to fix the data integrity, + which might be impossible if you cannot recover the original value. + +Also, you might observe a workaround for this concern by setting an offset in EE's values. +For example, this example sets `1000` as the offset: + +```ruby +module EE + module PipelineEnums + override :failure_reasons + def failure_reasons + super.merge(activity_limit_exceeded: 1_000, size_limit_exceeded: 1_001) + end + end +end +``` + +This looks working as a workaround, however, this approach has some donwside that: + +- Features could move from EE to FOSS or vice versa. Therefore, the offset might be mixed between FOSS and EE in the future. + e.g. When you move `activity_limit_exceeded` to FOSS, you'll see `{ unknown_failure: 0, config_error: 1, activity_limit_exceeded: 1_000 }`. +- The integer column for the `enum` is likely created [as `SMALLINT`](#creating-enums). + Therefore, you need to be careful of that the offset doesn't exceed the maximum value of 2 bytes integer. + +As a conclusion, you should define all of the key/value pairs in FOSS. +For example, you can simply write the following code in the above case: + +```ruby +class Pipeline < ApplicationRecord + enum failure_reason: { + unknown_failure: 0, + config_error: 1, + activity_limit_exceeded: 2 + } +end +``` diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index e090281f2bf..eda9e1e21cc 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -65,7 +65,7 @@ First, be aware of GitLab's [commitment to dogfooding](https://about.gitlab.com/ The code we write for Danger is GitLab-specific, and it **may not** be most appropriate place to implement functionality that addresses a need we encounter. Our users, customers, and even our own satellite projects, such as [Gitaly](https://gitlab.com/gitlab-org/gitaly), -often face similar challenges, after all. Think about how you could fulfil the +often face similar challenges, after all. Think about how you could fulfill the same need while ensuring everyone can benefit from the work, and do that instead if you can. @@ -145,7 +145,7 @@ at GitLab so far: fork. That way the danger comments will be made from CI using that API token instead. Making the variable - [masked](../ci/variables/README.md#masked-variables) will make sure + [masked](../ci/variables/README.md#mask-a-custom-variable) will make sure it doesn't show up in the job logs. The variable cannot be - [protected](../ci/variables/README.md#protected-environment-variables), + [protected](../ci/variables/README.md#protect-a-custom-variable), as it needs to be present for all feature branches. diff --git a/doc/development/database/add_foreign_key_to_existing_column.md b/doc/development/database/add_foreign_key_to_existing_column.md index e08f0a3bd1e..b8817eddeec 100644 --- a/doc/development/database/add_foreign_key_to_existing_column.md +++ b/doc/development/database/add_foreign_key_to_existing_column.md @@ -64,7 +64,7 @@ class AddNotValidForeignKeyToEmailsUser < ActiveRecord::Migration[5.2] def up # safe to use: it requires short lock on the table since we don't validate the foreign key - add_foreign_key :emails, :users, on_delete: :cascade, validate: false # rubocop:disable Migration/AddConcurrentForeignKey + add_foreign_key :emails, :users, on_delete: :cascade, validate: false end def down diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index 46cb869fea3..3753baa3c63 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -5,27 +5,33 @@ run into some head-banging database problems. An easy first step is to search for your error in Slack, or search for `GitLab <my error>` with Google. ---- +Available `RAILS_ENV`: -Available `RAILS_ENV` - -- `production` (generally not for your main GDK db, but you may need this for e.g. Omnibus) -- `development` (this is your main GDK db) -- `test` (used for tests like rspec) +- `production` (generally not for your main GDK database, but you may need this for other installations such as Omnibus). +- `development` (this is your main GDK db). +- `test` (used for tests like RSpec). ## Nuke everything and start over -If you just want to delete everything and start over with an empty DB (~1 minute): +If you just want to delete everything and start over with an empty DB (approximately 1 minute): -- `bundle exec rake db:reset RAILS_ENV=development` +```shell +bundle exec rake db:reset RAILS_ENV=development +``` -If you just want to delete everything and start over with dummy data (~4 minutes). This also does `db:reset` and runs DB-specific migrations: +If you just want to delete everything and start over with dummy data (approximately 4 minutes). This +also does `db:reset` and runs DB-specific migrations: -- `bundle exec rake dev:setup RAILS_ENV=development` +```shell +bundle exec rake dev:setup RAILS_ENV=development +``` -If your test DB is giving you problems, it is safe to nuke it because it doesn't contain important data: +If your test DB is giving you problems, it is safe to nuke it because it doesn't contain important +data: -- `bundle exec rake db:reset RAILS_ENV=test` +```shell +bundle exec rake db:reset RAILS_ENV=test +``` ## Migration wrangling diff --git a/doc/development/database_review.md b/doc/development/database_review.md index f2db0ab4fd5..aa7ebb3756f 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -74,12 +74,12 @@ the following preparations into account. #### Preparation when adding migrations -- Ensure `db/structure.sql` is updated. +- Ensure `db/structure.sql` is updated as [documented](migration_style_guide.md#schema-changes). - Make migrations reversible by using the `change` method or include a `down` method when using `up`. - Include either a rollback procedure or describe how to rollback changes. -- Add the output of both migrating and rolling back for all migrations into the MR description - - Ensure the down method reverts the changes in `db/structure.sql` - - Update the migration output whenever you modify the migrations during the review process +- Add the output of both migrating and rolling back for all migrations into the MR description. + - Ensure the down method reverts the changes in `db/structure.sql`. + - Update the migration output whenever you modify the migrations during the review process. - Add tests for the migration in `spec/migrations` if necessary. See [Testing Rails migrations at GitLab](testing_guide/testing_migrations_guide.md) for more details. - When [high-traffic](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/migration_helpers.rb#L12) tables are involved in the migration, use the [`with_lock_retries`](migration_style_guide.md#retry-mechanism-when-acquiring-database-locks) helper method. Review the relevant [examples in our documentation](migration_style_guide.md#examples) for use cases and solutions. - Ensure RuboCop checks are not disabled unless there's a valid reason to. diff --git a/doc/development/diffs.md b/doc/development/diffs.md index 3f855d90756..e065e0acc6f 100644 --- a/doc/development/diffs.md +++ b/doc/development/diffs.md @@ -8,12 +8,7 @@ Currently we rely on different sources to present diffs, these include: ## Deep Dive -In January 2019, Oswaldo Ferreira hosted a [Deep Dive] on GitLab's Diffs and Commenting on Diffs functionality to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube], and the slides on [Google Slides] and in [PDF]. Everything covered in this deep dive was accurate as of GitLab 11.7, and while specific details may have changed since then, it should still serve as a good introduction. - -[Deep Dive]: https://gitlab.com/gitlab-org/create-stage/issues/1 -[recording on YouTube]: https://www.youtube.com/watch?v=K6G3gMcFyek -[Google Slides]: https://docs.google.com/presentation/d/1bGutFH2AT3bxOPZuLMGl1ANWHqFnrxwQwjiwAZkF-TU/edit -[PDF]: https://gitlab.com/gitlab-org/create-stage/uploads/b5ad2f336e0afcfe0f99db0af0ccc71a/Create_Deep_Dive__Diffs_and_commenting_on_diffs.pdf +In January 2019, Oswaldo Ferreira hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on GitLab's Diffs and Commenting on Diffs functionality to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=K6G3gMcFyek), and the slides on [Google Slides](https://docs.google.com/presentation/d/1bGutFH2AT3bxOPZuLMGl1ANWHqFnrxwQwjiwAZkF-TU/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/b5ad2f336e0afcfe0f99db0af0ccc71a/). Everything covered in this deep dive was accurate as of GitLab 11.7, and while specific details may have changed since then, it should still serve as a good introduction. ## Architecture overview diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md new file mode 100644 index 00000000000..373c5a4ee7d --- /dev/null +++ b/doc/development/documentation/feature_flags.md @@ -0,0 +1,188 @@ +--- +type: reference +description: "GitLab development - how to document features deployed behind feature flags" +--- + +# Document features deployed behind feature flags + +GitLab uses [Feature Flags](../feature_flags/index.md) to strategically roll +out the deployment of its own features. The way we document a feature behind a +feature flag depends on its state (enabled or disabled). When the state +changes, the developer who made the change **must update the documentation** +accordingly. + +## Criteria + +According to the process of [deploying GitLab features behind feature flags](../feature_flags/process.md): + +> - _By default, feature flags should be off._ +> - _Feature flags should remain in the codebase for a short period as possible to reduce the need for feature flag accounting._ +> - _In order to build a final release and present the feature for self-managed users, the feature flag should be at least defaulted to on._ + +See how to document them below, according to the state of the flag: + +- [Features disabled by default](#features-disabled-by-default). +- [Features that became enabled by default](#features-that-became-enabled-by-default). +- [Features directly enabled by default](#features-directly-enabled-by-default). +- [Features with the feature flag removed](#features-with-flag-removed). + +NOTE: **Note:** +The [`**(CORE ONLY)**`](styleguide.md#product-badges) badge or equivalent for +the feature's tier should be added to the line and heading that refers to +enabling/disabling feature flags as Admin access is required to do so, +therefore, it indicates that it cannot be done by regular users of GitLab.com. + +### Features disabled by default + +For features disabled by default, if they cannot be used yet due to lack of +completeness, or if they're still under internal evaluation (for example, for +performance implications) do **not document them**: add (or merge) the docs +only when the feature is safe and ready to use and test by end users. + +For feature flags disabled by default, if they can be used by end users: + +- Say that it's disabled by default. +- Say whether it's enabled on GitLab.com. +- Say whether it's recommended for production use. +- Document how to enable and disable it. + +For example, for a feature disabled by default, disabled on GitLab.com, and +not ready for production use: + +````markdown +# Feature Name + +> - [Introduced](link-to-issue) in GitLab 12.0. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#anchor-to-section). **(CORE ONLY)** + +(...) + +### Enable or disable <Feature Name> **(CORE ONLY)** + +<Feature Name> is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../path/to/administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:<feature flag>) +``` + +To disable it: + +```ruby +Feature.disable(:<feature flag>) +``` +```` + +Adjust the blurb according to the state of the feature you're documenting. + +### Features that became enabled by default + +For features that became enabled by default: + +- Say that it became enabled by default. +- Say whether it's enabled on GitLab.com. +- Say whether it's recommended for production use. +- Document how to disable and enable it. + +For example, for a feature initially deployed disabled by default, that became enabled by default, that is enabled on GitLab.com, and ready for production use: + +````markdown +# Feature Name + +> - [Introduced](link-to-issue) in GitLab 12.0. +> - It was deployed behind a feature flag, disabled by default. +> - [Became enabled by default](link-to-issue) on GitLab 12.1. +> - It's enabled on GitLab.com. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#anchor-to-section). **(CORE ONLY)** + +(...) + +### Enable or disable <Feature Name> **(CORE ONLY)** + +<Feature Name> is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](..path/to/administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:<feature flag>) +``` + +To enable it: + +```ruby +Feature.enable(:<feature flag>) +``` +```` + +Adjust the blurb according to the state of the feature you're documenting. + +### Features directly enabled by default + +For features enabled by default: + +- Say it's enabled by default. +- Say whether it's enabled on GitLab.com. +- Say whether it's recommended for production use. +- Document how to disable and enable it. + +For example, for a feature enabled by default, enabled on GitLab.com, and ready for production use: + +````markdown +# Feature Name + +> - [Introduced](link-to-issue) in GitLab 12.0. +> - It's deployed behind a feature flag, enabled by default. +> - It's enabled on GitLab.com. +> - It's recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#anchor-to-section). **(CORE ONLY)** + +(...) + +### Enable or disable <Feature Name> **(CORE ONLY)** + +<Feature Name> is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](..path/to/administration/feature_flags.md) +can opt to disable it for your instance. + +To disable it: + +```ruby +Feature.disable(:<feature flag>) +``` + +To enable it: + +```ruby +Feature.enable(:<feature flag>) +``` +```` + +Adjust the blurb according to the state of the feature you're documenting. + +### Features with flag removed + +Once the feature is ready and the flag has been removed, clean up the +documentation. Remove the feature flag mention keeping only a note that +mentions the flag in the version history notes: + +````markdown +# Feature Name + +> - [Introduced](link-to-issue) in GitLab 12.0. +> - [Feature flag removed](link-to-issue) in GitLab 12.2. + +(...) + +```` diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index 7a0e187b70a..256d3f5d86c 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -6,13 +6,14 @@ description: Learn how to contribute to GitLab Documentation. GitLab's documentation is [intended as the single source of truth (SSOT)](https://about.gitlab.com/handbook/documentation/) for information about how to configure, use, and troubleshoot GitLab. The documentation contains use cases and usage instructions for every GitLab feature, organized by product area and subject. This includes topics and workflows that span multiple GitLab features, and the use of GitLab with other applications. -In addition to this page, the following resources can help you craft and contribute documentation: +In addition to this page, the following resources can help you craft and contribute to documentation: - [Style Guide](styleguide.md) - What belongs in the docs, language guidelines, Markdown standards to follow, links, and more. - [Structure and template](structure.md) - Learn the typical parts of a doc page and how to write each one. - [Documentation process](workflow.md). - [Markdown Guide](../../user/markdown.md) - A reference for all Markdown syntax supported by GitLab. - [Site architecture](site_architecture/index.md) - How <https://docs.gitlab.com> is built. +- [Documentation for feature flags](feature_flags.md) - How to write and update documentation for GitLab features deployed behind feature flags. ## Source files and rendered web locations @@ -51,7 +52,7 @@ docs-only merge requests using the following guide: [Contributions to GitLab docs](workflow.md) are welcome from the entire GitLab community. -To ensure that GitLab docs are current, there are special processes and responsibilities for all [feature changes](feature-change-workflow.md)—i.e. development work that impacts the appearance, usage, or administration of a feature. +To ensure that GitLab docs are current, there are special processes and responsibilities for all [feature changes](feature-change-workflow.md), that is development work that impacts the appearance, usage, or administration of a feature. However, anyone can contribute [documentation improvements](improvement-workflow.md) that are not associated with a feature change. For example, adding a new doc on how to accomplish a use case that's already possible with GitLab or with third-party tools and GitLab. @@ -76,13 +77,24 @@ whether the move is necessary), and ensure that a technical writer reviews this change prior to merging. If you indeed need to change a document's location, do not remove the old -document, but instead replace all of its content with a new line: +document, but instead replace all of its content with the following: -```md -This document was moved to [another location](path/to/new_doc.md). +```markdown +--- +redirect_to: '../path/to/file/index.md' +--- + +This document was moved to [another location](../path/to/file/index.md). ``` -where `path/to/new_doc.md` is the relative path to the root directory `doc/`. +Where `../path/to/file/index.md` is usually the relative path to the old document. + +The `redirect_to` variable supports both full and relative URLs, for example +`https://docs.gitlab.com/ee/path/to/file.html`, `../path/to/file.html`, `path/to/file.md`. +It ensures that the redirect will work for <https://docs.gitlab.com> and any `*.md` paths +will be compiled to `*.html`. +The new line underneath the front matter informs the user that the document +changed location and is useful for someone that browses that file from the repository. For example, if you move `doc/workflow/lfs/index.md` to `doc/administration/lfs.md`, then the steps would be: @@ -90,13 +102,17 @@ For example, if you move `doc/workflow/lfs/index.md` to 1. Copy `doc/workflow/lfs/index.md` to `doc/administration/lfs.md` 1. Replace the contents of `doc/workflow/lfs/index.md` with: - ```md + ```markdown + --- + redirect_to: '../../administration/lfs.md' + --- + This document was moved to [another location](../../administration/lfs.md). ``` 1. Find and replace any occurrences of the old location with the new one. - A quick way to find them is to use `git grep`. First go to the root directory - where you cloned the `gitlab` repository and then do: + A quick way to find them is to use `git grep` on the repository you changed + the file from: ```shell git grep -n "workflow/lfs/lfs_administration" @@ -123,24 +139,6 @@ Things to note: built-in help page, that's why we omit it in `git grep`. - Use the checklist on the "Change documentation location" MR description template. -### Alternative redirection method - -You can also replace the content -of the old file with a frontmatter containing a redirect link: - -```yaml ---- -redirect_to: '../path/to/file/README.md' ---- -``` - -It supports both full and relative URLs, e.g. `https://docs.gitlab.com/ee/path/to/file.html`, `../path/to/file.html`, `path/to/file.md`. Note that any `*.md` paths will be compiled to `*.html`. - -NOTE: **Note:** -This redirection method will not provide a redirect fallback on GitLab `/help`. When using -it, make sure to add a link to the new page on the doc, otherwise it's a dead end for users that -land on the doc via `/help`. - ### Redirections for pages with Disqus comments If the documentation page being relocated already has Disqus comments, @@ -150,12 +148,12 @@ Disqus uses an identifier per page, and for <https://docs.gitlab.com>, the page is configured to be the page URL. Therefore, when we change the document location, we need to preserve the old URL as the same Disqus identifier. -To do that, add to the frontmatter the variable `disqus_identifier`, -using the old URL as value. For example, let's say I moved the document +To do that, add to the front matter the variable `disqus_identifier`, +using the old URL as value. For example, let's say we moved the document available under `https://docs.gitlab.com/my-old-location/README.html` to a new location, `https://docs.gitlab.com/my-new-location/index.html`. -Into the **new document** frontmatter add the following: +Into the **new document** front matter, we add the following: ```yaml --- @@ -173,8 +171,8 @@ Before getting started, make sure you read the introductory section [documentation workflow](workflow.md). - Use the current [merge request description template](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/merge_request_templates/Documentation.md) -- Label the MR `Documentation` -- Assign the correct milestone (see note below) +- Label the MR `Documentation` (can only be done by people with `developer` access, for example, GitLab team members) +- Assign the correct milestone per note below (can only be done by people with `developer` access, for example, GitLab team members) Documentation will be merged if it is an improvement on existing content, represents a good-faith effort to follow the template and style standards, @@ -185,7 +183,7 @@ in a follow-up MR or issue. NOTE: **Note:** If the release version you want to add the documentation to has already been -frozen or released, use the label `Pick into X.Y` to get it merged into +frozen or released, use the label `~"Pick into X.Y"` to get it merged into the correct release. Avoid picking into a past release as much as you can, as it increases the work of the release managers. @@ -205,7 +203,7 @@ to the MR. For example, let's say your merge request has a milestone set to 11.3, which will be released on 2018-09-22. If it gets merged on 2018-09-15, it will be available online on 2018-09-15, but, as the feature freeze date has passed, if -the MR does not have a "pick into 11.3" label, the milestone has to be changed +the MR does not have a `~"Pick into 11.3"` label, the milestone has to be changed to 11.4 and it will be shipped with all GitLab packages only on 2018-10-22, with GitLab 11.4. Meaning, it will only be available under `/help` from GitLab 11.4 onward, but available on <https://docs.gitlab.com/> on the same day it was merged. @@ -277,7 +275,7 @@ You can combine one or more of the following: ### GitLab `/help` tests -Several [rspec tests](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/features/help_pages_spec.rb) +Several [RSpec tests](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/features/help_pages_spec.rb) are run to ensure GitLab documentation renders and works correctly. In particular, that [main docs landing page](../../README.md) will work correctly from `/help`. For example, [GitLab.com's `/help`](https://gitlab.com/help). @@ -300,8 +298,8 @@ To preview your changes to documentation locally, follow this The live preview is currently enabled for the following projects: -- <https://gitlab.com/gitlab-org/gitlab> -- <https://gitlab.com/gitlab-org/gitlab-runner> +- [`gitlab`](https://gitlab.com/gitlab-org/gitlab) +- [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner) If your merge request has docs changes, you can use the manual `review-docs-deploy` job to deploy the docs review app for your merge request. @@ -407,8 +405,6 @@ merge request with new or changed docs is submitted, are: - [`internal_anchors`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/docs.gitlab-ci.yml#L69) checks that all internal anchors (ex: `[link](../index.md#internal_anchor)`) are valid. -- If any code or the `doc/README.md` file is changed, a full pipeline will run, which - runs tests for [`/help`](#gitlab-help-tests). ### Running tests @@ -489,14 +485,14 @@ markdownlint can be used [on the command line](https://github.com/igorshubovych/ either on a single Markdown file or on all Markdown files in a project. For example, to run markdownlint on all documentation in the [`gitlab` project](https://gitlab.com/gitlab-org/gitlab), run the following commands from within your `gitlab` project root directory, which will -automatically detect the [`.markdownlint.json`](#markdownlint-configuration) config +automatically detect the [`.markdownlint.json`](#markdownlint-configuration) configuration file in the root of the project, and test all files in `/doc` and its subdirectories: ```shell markdownlint 'doc/**/*.md' ``` -If you wish to use a different config file, use the `-c` flag: +If you wish to use a different configuration file, use the `-c` flag: ```shell markdownlint -c <config-file-name> 'doc/**/*.md' @@ -510,7 +506,7 @@ such as: - [Atom](https://atom.io/packages/linter-node-markdownlint) It is best to use the [same configuration file](#markdownlint-configuration) as what -is in use in the four repos that are the sources for <https://docs.gitlab.com>. Each +is in use in the four repositories that are the sources for <https://docs.gitlab.com>. Each plugin/extension has different requirements regarding the configuration file, which is explained in each editor's docs. @@ -519,12 +515,12 @@ is explained in each editor's docs. Each formatting issue that markdownlint checks has an associated [rule](https://github.com/DavidAnson/markdownlint/blob/master/doc/Rules.md#rules). These rules are configured in the `.markdownlint.json` files located in the root of -four repos that are the sources for <https://docs.gitlab.com>: +four repositories that are the sources for <https://docs.gitlab.com>: -- <https://gitlab.com/gitlab-org/gitlab/blob/master/.markdownlint.json> -- <https://gitlab.com/gitlab-org/gitlab-runner/blob/master/.markdownlint.json> -- <https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/.markdownlint.json> -- <https://gitlab.com/charts/gitlab/blob/master/.markdownlint.json> +- [`gitlab`](https://gitlab.com/gitlab-org/gitlab/blob/master/.markdownlint.json) +- [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner/blob/master/.markdownlint.json) +- [`omnibus-gitlab`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/.markdownlint.json) +- [`charts`](https://gitlab.com/charts/gitlab/blob/master/.markdownlint.json) By default all rules are enabled, so the configuration file is used to disable unwanted rules, and also to configure optional parameters for enabled rules as needed. You can @@ -534,7 +530,7 @@ on or off when markdownlint was enabled on the docs. #### Vale -[Vale](https://errata-ai.github.io/vale/) is a grammar, style, and word usage linter +[Vale](https://errata-ai.gitbook.io/vale/) is a grammar, style, and word usage linter for the English language. Vale's configuration is stored in the [`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/blob/master/.vale.ini) file located in the root directory of the [GitLab repository](https://gitlab.com/gitlab-org/gitlab). @@ -554,10 +550,28 @@ You can also [configure the text editor of your choice](https://errata-ai.github.io/vale/#local-use-by-a-single-writer) to display the results. +Vale's test results [are displayed](#testing) in CI pipelines. + +##### Disable a Vale test + +You can disable a specific Vale linting rule or all Vale linting rules for any portion of a document: + +- To disable a specific rule, add a `<!-- vale gitlab.rulename = NO -->` tag + before the text, and a `<!-- vale gitlab.rulename = YES -->` tag after the text, + replacing `rulename` with the filename of a test in the [GitLab styles](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/.linting/vale/styles/gitlab) directory. +- To disable all Vale linting rules, add a `<!-- vale off -->` tag before the text, + and a `<!-- vale on -->` tag after the text. + +Whenever possible, exclude only the problematic rule and line(s). +In some cases, such as list items, you may need to disable linting for the entire +list until ["Ignore comments are not honored in a Markdown file"](https://github.com/errata-ai/vale/issues/175) is resolved. + +For more information, see [Vale's documentation](https://errata-ai.gitbook.io/vale/getting-started/markup#markup-based-configuration). + ## Danger Bot GitLab uses [Danger](https://github.com/danger/danger) for some elements in code review. For docs changes in merge requests, whenever a change to files under `/doc` is made, Danger Bot leaves a comment with further instructions about the documentation -process. This is configured in the Dangerfile in the GitLab repo under +process. This is configured in the `Dangerfile` in the GitLab repository under [/danger/documentation/](https://gitlab.com/gitlab-org/gitlab/tree/master/danger/documentation). diff --git a/doc/development/documentation/site_architecture/release_process.md b/doc/development/documentation/site_architecture/release_process.md index a8da565124b..13d6540fa35 100644 --- a/doc/development/documentation/site_architecture/release_process.md +++ b/doc/development/documentation/site_architecture/release_process.md @@ -20,7 +20,7 @@ Although build images are built automatically via GitLab CI/CD, you can build and tag all tooling images locally: 1. Make sure you have [Docker installed](https://docs.docker.com/install/). -1. Make sure you're on the `dockerfiles/` directory of the `gitlab-docs` repo. +1. Make sure you're in the `dockerfiles/` directory of the `gitlab-docs` repository. 1. Build the images: ```shell @@ -46,7 +46,7 @@ products, we need to add a 1. Check that there is a [stable branch created](https://gitlab.com/gitlab-org/charts/gitlab/-/branches) for the new chart version. If you're unsure or can't find it, drop a line in the `#g_delivery` channel. -1. Make sure you're on the root path of the `gitlab-docs` repo. +1. Make sure you're in the root path of the `gitlab-docs` repository. 1. Open `content/_data/chart_versions.yaml` and add the new stable branch version using the version mapping. Note that only the `major.minor` version is needed. 1. Create a new merge request and merge it. @@ -61,14 +61,14 @@ this first step. The single docs version must be created before the release merge request, but this needs to happen when the stable branches for all products have been created. -1. Make sure you're on the root path of the `gitlab-docs` repo. +1. Make sure you're in the root path of the `gitlab-docs` repository. 1. Make sure your `master` is updated: ```shell git pull origin master ``` -1. Run the raketask to create the single version: +1. Run the Rake task to create the single version: ```shell ./bin/rake "release:single[12.0]" @@ -109,7 +109,7 @@ Visit `http://localhost:4000/12.0/` to see if everything works correctly. Now it's time to create the monthly release merge request that adds the new version and rotates the old one: -1. Make sure you're on the root path of the `gitlab-docs` repo. +1. Make sure you're in the root path of the `gitlab-docs` repository. 1. Create a branch `release-X-Y`: ```shell @@ -158,10 +158,10 @@ the dropdown are included in the unmerged `release-X-Y` branch. The content of `content/_data/versions.yaml` needs to change for all online versions: -1. Run the raketask that will create all the respective merge requests needed to +1. Run the Rake task that will create all the respective merge requests needed to update the dropdowns and will be set to automatically be merged when their pipelines succeed. The `release-X-Y` branch needs to be present locally, - otherwise the raketask will fail: + otherwise the Rake task will fail: ```shell ./bin/rake release:dropdowns diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index 84ba47eba78..d19383bee27 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -20,7 +20,7 @@ Every feature or use case document should include the following content in the f with exceptions and details noted below and in the template included on this page. - **Title**: Top-level heading with the feature name, or a use case name, which would start with - a verb, like Configuring, Enabling, etc. + a verb, like Configuring, Enabling, and so on. - **Introduction**: A couple sentences about the subject matter and what's to be found on this page. Describe what the feature or topic is, what it does, and in what context it should be used. There is no need to add a title called "Introduction" or "Overview," because people rarely @@ -41,7 +41,7 @@ and other logical divisions such as pre- and post-deployment steps. To start a new document, respect the file tree and file name guidelines, as well as the style guidelines. Use the following template: -```md +```markdown <!--Follow the Style Guide when working on this document. https://docs.gitlab.com/ee/development/documentation/styleguide.html When done, remove all of this commented-out text, except a commented-out Troubleshooting section, which, if empty, can be left in place to encourage future use.--> @@ -130,7 +130,7 @@ Notes: ## Help and feedback section The "help and feedback" section (introduced by [!319](https://gitlab.com/gitlab-org/gitlab-docs/-/merge_requests/319)) displayed at the end of each document -can be omitted from the doc by adding a key into the its frontmatter: +can be omitted from the doc by adding a key into the its front matter: ```yaml --- @@ -148,7 +148,7 @@ We also have integrated the docs site with Disqus (introduced by allowing our users to post comments. To omit only the comments from the feedback section, use the following -key on the frontmatter: +key on the front matter: ```yaml --- @@ -159,7 +159,7 @@ comments: false We are only hiding comments in main index pages, such as [the main documentation index](../../README.md), since its content is too broad to comment on. Before omitting Disqus, you must check with a technical writer. -Note that once `feedback: false` is added to the frontmatter, it will automatically omit +Note that once `feedback: false` is added to the front matter, it will automatically omit Disqus, therefore, don't add both keys to the same document. The click events in the feedback section are tracked with Google Tag Manager. The diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 0007f6d6e2f..6d146051e13 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -17,9 +17,9 @@ that apply to all GitLab content, not just documentation. ### Why a single source of truth -The documentation of GitLab products and features is the SSOT for all information related to implementation, usage, and troubleshooting. It evolves continually, in keeping with new products and features, and with improvements for clarity, accuracy, and completeness. +The documentation of GitLab products and features is the SSOT for all information related to implementation, usage, and troubleshooting. It evolves continuously, in keeping with new products and features, and with improvements for clarity, accuracy, and completeness. -This policy prevents information silos, ensuring that it remains easy to find information about GitLab products. +This policy prevents information silos, making it easier to find information about GitLab products. It also informs decisions about the kinds of content we include in our documentation. @@ -46,12 +46,12 @@ In the software industry, it is a best practice to organize documentation in dif 1. Explanation 1. Reference (for example, a glossary) -At GitLab, we have so many product changes in our monthly releases that we can't afford to continually update multiple types of information. +At GitLab, we have so many product changes in our monthly releases that we can't afford to continuously update multiple types of information. If we have multiple types, the information will become outdated. Therefore, we have a [single template](structure.md) for documentation. We currently do not distinguish specific document types, although we are open to reconsidering this policy once the documentation has reached a future stage of maturity and quality. If you are reading this, then despite our -continual improvement efforts, that point hasn't been reached. +continuous improvement efforts, that point hasn't been reached. ### Link instead of summarize @@ -61,7 +61,7 @@ Instead, link to the SSOT and explain why it is important to consume the informa ### Organize by topic, not by type -Beyond top-level audience-type folders (for example, `administration`), we organize content by topic, not by type, so that it can be located as easily as possible within the single-source-of-truth (SSOT) section for the subject matter. +Beyond top-level audience-type folders (for example, `administration`), we organize content by topic, not by type, so it can be located as easily as possible within the single-source-of-truth (SSOT) section for the subject matter. For example, do not create groupings of similar media types. For example: @@ -76,7 +76,7 @@ and cross-link between any related content. ### Docs-first methodology -We employ a **docs-first methodology** to help ensure that the docs remain a complete and trusted resource, and to make communicating about the use of GitLab more efficient. +We employ a **docs-first methodology** to help ensure the docs remain a complete and trusted resource, and to make communicating about the use of GitLab more efficient. - If the answer to a question exists in documentation, share the link to the docs instead of rephrasing the information. - When you encounter new information not available in GitLab’s documentation (for example, when working on a support case or testing a feature), your first step should be to create a merge request (MR) to add this information to the docs. You can then share the MR in order to communicate this information. @@ -129,13 +129,13 @@ correctly, but is not the current standard for GitLab documentation). A rule that could cause confusion is `MD044/proper-names`, as it might not be immediately clear what caused markdownlint to fail, or how to correct the failure. This rule checks a list of known words, listed in the `.markdownlint.json` file in each project, -to verify that proper capitalization and backticks are used. Words in backticks will +to verify proper use of capitalization and backticks. Words in backticks will be ignored by markdownlint. In general, product names should follow the exact capitalization of the official names of the products, protocols, and so on. -Some examples that will fail if incorrect capitalization is used: +Some examples fail if incorrect capitalization is used: - MinIO (needs capital `IO`) - NGINX (needs all capitals) @@ -248,10 +248,6 @@ GitLab documentation should be clear and easy to understand. - Be clear, concise, and stick to the goal of the documentation. - Write in US English with US grammar. - Use inclusive language. -- Avoid jargon. -- Avoid uncommon words. -- Don't write in the first person singular. - - Instead of "I" or "me," use "we," "you," "us," or "one." ### Point of view @@ -285,21 +281,53 @@ because it’s friendly and easy to understand. Some features are also objects. For example, "GitLab's Merge Requests support X" and "Create a new merge request for Z." +### Language to avoid + +When creating documentation, limit or avoid the use of the following verb +tenses, words, and phrases: + +- Avoid jargon. +- Avoid uncommon words. +- Don't write in the first person singular. + - Instead of "I" or "me," use "we," "you," "us," or "one." + - When possible, stay user focused by writing in the second person ("you" or + the imperative). +- Don't overuse "that". In many cases, you can remove "that" from a sentence + and improve readability. - Avoid use of the future tense: - - Instead of "after you execute this command, GitLab will display the result", use "after you execute this command, GitLab displays the result". - - Only use the future tense to convey when the action or result will actually occur at a future time. -- Do not use slashes to clump different words together or as a replacement for the word "or": - - Instead of "and/or," consider using "or," or use another sensible construction. - - Other examples include "clone/fetch," author/assignee," and "namespace/repository name." Break apart any such instances in an appropriate way. - - Exceptions to this rule include commonly accepted technical terms such as CI/CD, TCP/IP, and so on. -- Do not use "may" and "might" interchangeably: - - Use "might" to indicate the probability of something occurring. "If you skip this step, the import process might fail." - - Use "may" to indicate giving permission for someone to do something, or consider using "can" instead. "You may select either option on this screen." Or, "you can select either option on this screen." -- We discourage use of Latin abbreviations, such as "e.g.," "i.e.," or "etc.," -as even native users of English might misunderstand them. + - Instead of "after you execute this command, GitLab will display the + result", use "after you execute this command, GitLab displays the result". + - Only use the future tense to convey when the action or result will actually + occur at a future time. +- Don't use slashes to clump different words together or as a replacement for + the word "or": + - Instead of "and/or," consider using "or," or use another sensible + construction. + - Other examples include "clone/fetch," author/assignee," and + "namespace/repository name." Break apart any such instances in an + appropriate way. + - Exceptions to this rule include commonly accepted technical terms, such as + CI/CD and TCP/IP. +- <!-- vale gitlab.LatinTerms = NO --> + We discourage use of Latin abbreviations, such as "e.g.," "i.e.," or "etc.," + as even native users of English might misunderstand them. - Instead of "i.e.," use "that is." - Instead of "e.g.," use "for example," "such as," "for instance," or "like." - - Instead of "etc.," either use "and so on" or consider editing it out, since it can be vague. + - Instead of "etc.," either use "and so on" or consider editing it out, since + it can be vague. + <!-- vale gitlab.rulename = NO --> +- Avoid using the word *currently* when talking about the product or its + features. The documentation describes the product as it is, and not as it + will be at some indeterminate point in the future. + +### Word usage clarifications + +- Don't use "may" and "might" interchangeably: + - Use "might" to indicate the probability of something occurring. "If you + skip this step, the import process might fail." + - Use "may" to indicate giving permission for someone to do something, or + consider using "can" instead. "You may select either option on this + screen." Or, "You can select either option on this screen." ### Contractions @@ -353,6 +381,8 @@ as even native users of English might misunderstand them. | Requests to localhost are not allowed | Requests to localhost aren't allowed | | Specified URL cannot be used | Specified URL can't be used | +<!-- vale on --> + ## Text - [Write in Markdown](#markdown). @@ -360,7 +390,7 @@ as even native users of English might misunderstand them. - Insert an empty line for new paragraphs. - Insert an empty line between different markups (for example, after every paragraph, header, list, and so on). Example: - ```md + ```markdown ## Header Paragraph. @@ -417,7 +447,7 @@ Only use ordered lists when their items describe a sequence of steps to follow. Do: -```md +```markdown These are the steps to do something: 1. First, do the first step. @@ -427,7 +457,7 @@ These are the steps to do something: Don't: -```md +```markdown This is a list of available features: 1. Feature 1 @@ -453,7 +483,7 @@ This is a list of available features: all with a period. - Separate list items from explanatory text with a colon (`:`). For example: - ```md + ```markdown The list is as follows: - First item: this explains the first item. @@ -511,7 +541,7 @@ In unordered lists (using `-`), this means two spaces for each level of indentat - Unordered list item 3 - ```text + ```plaintext a codeblock that will next inside list item 3 ``` @@ -534,7 +564,7 @@ For ordered lists, use three spaces for each level of indentation: 1. Ordered list item 3 - ```text + ```plaintext a codeblock that will next inside list item 3 ``` @@ -560,9 +590,47 @@ to mix types, that is also possible, as long as you don't mix items at the same - Unordered list item three. ``` +## Tables + +Tables should be used to describe complex information in a straightforward +manner. Note that in many cases, an unordered list is sufficient to describe a +list of items with a single, simple description per item. But, if you have data +that is best described by a matrix, tables are the best choice for use. + +### Creation guidelines + +Due to accessibility and scanability requirements, tables should not have any +empty cells. If there is no otherwise meaningful value for a cell, consider entering +*N/A* (for 'not applicable') or *none*. + +To help tables be easier to maintain, consider adding additional spaces to the +column widths to make them consistent. For example: + +```markdown +| App name | Description | Requirements | +|:---------|:---------------------|:---------------| +| App 1 | Description text 1. | Requirements 1 | +| App 2 | Description text 2. | None | +``` + +Consider installing a plugin or extension in your editor for formatting tables: + +- [Markdown Table Prettifier](https://marketplace.visualstudio.com/items?itemName=darkriszty.markdown-table-prettify) for Visual Studio Code +- [Markdown Table Formatter](https://packagecontrol.io/packages/Markdown%20Table%20Formatter) for Sublime Text +- [Markdown Table Formatter](https://atom.io/packages/markdown-table-formatter) for Atom + +### Feature tables + +When creating tables of lists of features (such as whether or not features are +available to certain roles on the [Permissions](../../user/permissions.md#project-members-permissions) +page), use the following phrases (based on the SVG icons): + +- *No*: **{dotted-circle}** No +- *Yes*: **{check-circle}** Yes + ## Quotes -Valid for Markdown content only, not for frontmatter entries: +Valid for Markdown content only, not for front matter entries: - Standard quotes: double quotes (`"`). Example: "This is wrapped in double quotes". - Quote within a quote: double quotes (`"`) wrap single quotes (`'`). Example: "I am 'quoting' something within a quote". @@ -724,15 +792,17 @@ Instead: Example: -```md +```markdown For more information, see the [confidential issue](../../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab-foss/issues/<issue_number>`. ``` ### Link to specific lines of code -When linking to specifics lines within a file, link to a commit instead of to the branch. +When linking to specific lines within a file, link to a commit instead of to the branch. Lines of code change through time, therefore, linking to a line by using the commit link -ensures the user lands on the line you're referring to. +ensures the user lands on the line you're referring to. The **Permalink** button, which is +available when viewing a file within a project, makes it easy to generate a link to the +most recent commit of the given file. - **Do:** `[link to line 3](https://gitlab.com/gitlab-org/gitlab/-/blob/11f17c56d8b7f0b752562d78a4298a3a95b5ce66/.gitlab/issue_templates/Feature%20proposal.md#L3)` - **Don't:** `[link to line 3](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20proposal.md#L3).` @@ -801,7 +871,7 @@ known tool is [`pngquant`](https://pngquant.org/), which is cross-platform and open source. Install it by visiting the official website and following the instructions for your OS. -GitLab has a [raketask](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/tasks/pngquant.rake) +GitLab has a [Rake task](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/tasks/pngquant.rake) that you can use to automate the process. In the root directory of your local copy of `https://gitlab.com/gitlab-org/gitlab`, run in a terminal: @@ -838,7 +908,7 @@ Do not upload videos to the product repositories. [Link](#link-to-video) or [emb To link out to a video, include a YouTube icon so that readers can quickly and easily scan the page for videos before reading: -```md +```markdown <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [Video Title](link-to-video). ``` @@ -965,7 +1035,7 @@ of language classes available. | `ruby` | Alias: `rb`. | | `shell` | Aliases: `bash` or `sh`. | | `sql` | | -| `toml` | Runner configuration examples, and other toml formatted configuration files. | +| `toml` | Runner configuration examples, and other TOML-formatted configuration files. | | `typescript` | Alias: `ts`. | | `xml` | | | `yaml` | Alias: `yml`. | @@ -1012,7 +1082,7 @@ This will ensure that the source Markdown remains readable and should help with The following are examples of source Markdown for menu items with their published output: -```md +```markdown 1. Go to **{home}** **Project overview > Details** 1. Go to **{doc-text}** **Repository > Branches** 1. Go to **{issues}** **Issues > List** @@ -1073,7 +1143,7 @@ of users. Weigh the costs of distracting users to whom the content is not relevant against the cost of users missing the content if it were not expressed as a note. -```md +```markdown NOTE: **Note:** This is something to note. ``` @@ -1085,7 +1155,7 @@ This is something to note. ### Tip -```md +```markdown TIP: **Tip:** This is a tip. ``` @@ -1097,7 +1167,7 @@ This is a tip. ### Caution -```md +```markdown CAUTION: **Caution:** This is something to be cautious about. ``` @@ -1109,7 +1179,7 @@ This is something to be cautious about. ### Danger -```md +```markdown DANGER: **Danger:** This is a breaking change, a bug, or something very important to note. ``` @@ -1123,7 +1193,7 @@ This is a breaking change, a bug, or something very important to note. For highlighting a text within a blue blockquote, use this format: -```md +```markdown > This is a blockquote. ``` @@ -1135,7 +1205,7 @@ If the text spans across multiple lines it's OK to split the line. For multiple paragraphs, use the symbol `>` before every line: -```md +```markdown > This is the first paragraph. > > This is the second paragraph. @@ -1216,7 +1286,7 @@ a helpful link back to how the feature was developed. to the following should be added immediately below the heading as a blockquote: - `> Introduced in GitLab 11.3.`. -- Whenever possible, version text should have a link to the issue, merge request, or epic that introduced the feature. +- Whenever possible, version text should have a link to the _completed_ issue, merge request, or epic that introduced the feature. An issue is preferred over a merge request, and a merge request is preferred over an epic. For example: - `> [Introduced](<link-to-issue>) in GitLab 11.3.`. @@ -1228,21 +1298,35 @@ a helpful link back to how the feature was developed. - If listing information for multiple version as a feature evolves, add the information to a block-quoted bullet list. For example: - ```md + ```markdown > - [Introduced](<link-to-issue>) in GitLab 11.3. > - Enabled by default in GitLab 11.4. ``` - If a feature is moved to another tier: - ```md + ```markdown > - [Introduced](<link-to-issue>) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.5. > - [Moved](<link-to-issue>) to [GitLab Starter](https://about.gitlab.com/pricing/) in 11.8. > - [Moved](<link-to-issue>) to GitLab Core in 12.0. ``` NOTE: **Note:** -Version text must be on its own line and surounded by blank lines to render correctly. +Version text must be on its own line and surrounded by blank lines to render correctly. + +### Versions in the past or future + +When describing functionality available in past or future versions, use: + +- **Earlier**, and not **older** or **before**. +- **Later**, and not **newer** or **after**. + +For example: + +- Available in GitLab 12.3 and earlier. +- Available in GitLab 12.4 and later. +- If using GitLab 11.4 or earlier, ... +- If using GitLab 10.6 or later, ... ### Importance of referencing GitLab versions and tiers @@ -1333,7 +1417,7 @@ avoid duplication, link to the special document that can be found in [`doc/administration/restart_gitlab.md`](../../administration/restart_gitlab.md). Usually the text will read like: -```md +```markdown Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) for the changes to take effect. ``` @@ -1369,7 +1453,7 @@ When there is a list of steps to perform, usually that entails editing the configuration file and reconfiguring/restarting GitLab. In such case, follow the style below as a guide: -````md +````markdown **For Omnibus installations** 1. Edit `/etc/gitlab/gitlab.rb`: @@ -1416,38 +1500,8 @@ can facilitate this by making sure the troubleshooting content addresses: ## Feature flags -Sometimes features are shipped with feature flags, either: - -- On by default, but providing the option to turn the feature off. -- Off by default, but providing the option to turn the feature on. - -When documenting feature flags for a feature, include: - -- Why a feature flag is necessary. Some of the reasons are - [outlined in the handbook](https://about.gitlab.com/handbook/product/#alpha-beta-ga). -- That administrative access is required to make a feature flag change. -- What to ask for when requesting a change to a feature flag's state. - -NOTE: **Note:** -The [Product Manager for the relevant group](https://about.gitlab.com/handbook/product/categories/#devops-stages) -must review and approve the addition or removal of any mentions of using feature flags before the doc change is merged. - -The following is sample text for adding feature flag documentation for a feature that is -off by default: - -````md -### Enabling the feature - -This feature comes with the `:feature_flag` feature flag disabled by default. In some cases, -this feature is incompatible with an old configuration. To turn on the feature, -ask a GitLab administrator with Rails console access to run the following command: - -```ruby -Feature.enable(:feature_flag) -``` -```` - -For guidance on developing with feature flags, see +Learn how to [document features deployed behind flags](feature_flags.md). +For guidance on developing GitLab with feature flags, see [Feature flags in development of GitLab](../feature_flags/index.md). ## API @@ -1475,7 +1529,7 @@ The following can be used as a template to get started: One or two sentence description of what endpoint does. -```text +```plaintext METHOD /endpoint ``` @@ -1537,7 +1591,7 @@ You can use the following fake tokens as examples. Use the following table headers to describe the methods. Attributes should always be in code blocks using backticks (`` ` ``). -```md +```markdown | Attribute | Type | Required | Description | |:----------|:-----|:---------|:------------| ``` @@ -1614,7 +1668,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "title= ``` The above example is run by and administrator and will add an SSH public key -titled `ssh-key` to user's account which has an id of 25. +titled `ssh-key` to user's account which has an ID of 25. #### Escape special characters diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index 01f528dcfa4..ab6200155bf 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -1,19 +1,20 @@ # Documentation process -The process for creating and maintaining GitLab product documentation depends on whether the -documentation is associated with: +The process for creating and maintaining GitLab product documentation allows +anyone to contribute a merge request or create an issue for GitLab's +documentation. -- [A new feature or feature enhancement](#for-a-product-change). - - Delivered for a specific milestone and associated with specific code changes. This documentation - has the highest priority. +NOTE: **Note:** +Documentation updates relating to new features or feature enhancements must +use the [feature workflow process](https://about.gitlab.com/handbook/engineering/ux/technical-writing/workflow/#for-a-product-change) described in the GitLab Handbook. -- [Changes outside a specific milestone](#for-all-other-documentation). +## Who updates the docs? - It is usually not associated with a specific code change and has a lower priority. +*Anyone* can contribute! You can create a merge request for documentation when: -Documentation is not usually required when a "backstage feature" is added or changed, and does not -directly affect the way that any user or administrator interacts with GitLab. +- You find errors or other room for improvement in existing documentation. +- You have an idea for all-new documentation that would help a GitLab user or administrator to + accomplish their work with GitLab. ## Documentation labels @@ -32,372 +33,17 @@ The following are also added by members of the Technical Writing team: `docs::` prefix. For example, `~docs::improvement`. - The `~Technical Writing` [team label](../contributing/issue_workflow.md#team-labels). -## For a product change - -This documentation is required for any new or changed feature and is: - -- Created or updated as part of feature development, almost always in the same merge request as the - feature code. Including documentation in the same merge request as the code eliminates the - possibility that code and documentation get out of sync. -- Required with the delivery of a feature for a specific milestone as part of GitLab's - [definition of done](../contributing/merge_request_workflow.md#definition-of-done). -- Often linked from the release post. - -### Roles and responsibilities - -Documentation for specific milestones involves the: - -- Developer of a feature or enhancement. -- Product Manager for the group delivering the new feature or feature enhancement. -- Technical Writer assigned to the group. - -Each role is described below. - -#### Developers - -Developers are the primary author of documentation for a feature or feature enhancement. They are -responsible for: - -- Developing initial content required for a feature. -- Liaising with their Product Manager to understand what documentation must be delivered, and when. -- Requesting technical reviews from other developers within their group. -- Requesting documentation reviews from the Technical Writer - [assigned to the DevOps stage group](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) - that is delivering the new feature or feature enhancements. - -TIP: **Tip:** -Community Contributors can ask for additional help from GitLab team members. - -##### Authoring - -Because the documentation is an essential part of the product, if a ~feature issue also contains the -~documentation label, you must ship the new or updated documentation with the code of the feature. - -Technical Writers are happy to help, as requested and planned on an issue-by-issue basis. - -For feature issues requiring documentation, follow the process below unless otherwise agreed with -the Product Manager and Technical Writer for a given issue: - -- Include any new and edited documentation, either in: - - The merge request introducing the code. - - A separate merge request raised around the same time. -- Use the [documentation requirements](#documentation-requirements) developed by the Product Manager - in the issue and discuss any further documentation plans or ideas as needed. - - If the new or changed documentation requires extensive collaboration or conversation, a - separate, linked issue can be used for the planning process. - -- Use the [Documentation guidelines](index.md), as well as other resources linked from there, - including: - - Documentation [Structure and template](structure.md) page. - - [Style Guide](styleguide.md). - - [Markdown Guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/). -- Contact the Technical Writer for the relevant [DevOps stage](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) - in your issue or merge request, or within `#docs` on GitLab Slack, if you: - - Need any help to choose the correct place for documentation. - - Want to discuss a documentation idea or outline. - - Want to request any other help. -- If you are working on documentation in a separate merge request, ensure the documentation is - merged as close as possible to the code merge. -- A policy for documenting [feature-flagged](../feature_flags/index.md) issues is forthcoming and you - are welcome to join the [discussion](https://gitlab.com/gitlab-org/gitlab/issues/26347). - -##### Reviews and merging - -Reviewers help ensure: - -- Accuracy. -- Clarity. -- Completeness. -- Adherence to: - - [Documentation requirements](#documentation-requirements) in the issue. - - [Documentation guidelines](index.md). - - [Style guide](styleguide.md). - -Prior to merging, documentation changes committed by the developer must be reviewed by: - -- The code reviewer for the merge request. This is known as a technical review. -- Optionally, others involved in the work such as other developers or the Product Manager. -- The Technical Writer for the DevOps stage group, except in exceptional circumstances where a - [post-merge review](#post-merge-reviews) can be requested. -- A maintainer of the project. - -#### Product Managers - -Product Managers are responsible for the [documentation requirements](#documentation-requirements) -for a feature or feature enhancement. They can also: - -- Liaise with the Technical Writer for discussion and collaboration. -- Review documentation themselves. - -For issues requiring any new or updated documentation, the Product Manager must: - -- Add the ~documentation label. -- Confirm or add the [documentation requirements](#documentation-requirements). -- Ensure the issue contains: - - Any new or updated feature name. - - Overview, description, and use cases when applicable (as required by the - [documentation structure and template](structure.md)). - -Everyone is encouraged to draft the documentation requirements in the issue. However, a Product -Manager will: - -- When the issue is assigned a release milestone, review and update the Documentation details. -- By the kickoff, finalize the documentation details. - -#### Technical Writers - -Technical Writers are responsible for: - -- Participating in issues discussions and reviewing MRs for the upcoming milestone. -- Reviewing documentation requirements in issues when called upon. -- Answering questions, and helping and providing advice throughout the authoring and editing - process. -- Reviewing all significant new and updated documentation content, whether before merge or after it - is merged. -- Assisting the developer and Product Manager with feature documentation delivery. - -##### Planning - -The Technical Writer: - -- Reviews their group's `~feature` issues that are part of the next milestone to get a sense of the - scope of content likely to be authored. -- Recommends the `~documentation` label on issues from that list which don't have it but should, or - inquires with the PM to determine if documentation is truly required. -- For `~direction` issues from that list, reads the full issue and reviews its Documentation - requirements section. Addresses any recommendations or questions with the PMs and others - collaborating on the issue in order to refine or expand the Documentation requirements. - -##### Collaboration - -By default, the developer will work on documentation changes independently, but -the developer, Product Manager, or Technical Writer can propose a broader collaboration for -any given issue. - -Additionally, Technical Writers are available for questions at any time. - -##### Review - -Technical Writers: - -- Provide non-blocking reviews of all documentation changes, before or after the change is merged. -- Confirm that the documentation is: - - Clear. - - Grammatically correct. - - Discoverable. - - Navigable. -- Ensures that the documentation avoids: - - Redundancy. - - Bad file locations. - - Typos. - - Broken links. - -The Technical Writer will review the documentation to check that the developer and -code reviewer have ensured: - -- Clarity. -- Appropriate location, making sure the documentation is in the correct directories (often - reflecting how the product is structured) and has the correct name. -- Syntax, typos, and broken links. -- Improvements to the content. -- Accordance with the: - - [Documentation Style Guide](styleguide.md). - - [Structure and Template](structure.md) doc. - -### When documentation is required - -Documentation [is required](../contributing/merge_request_workflow.md#definition-of-done) for a -milestone when: - -- A new or enhanced feature is shipped that impacts the user or administrator experience. -- There are changes to the UI or API. -- A process, workflow, or previously documented feature is changed. -- A feature is deprecated or removed. - -NOTE: **Note:** -Documentation refactoring unrelated to a feature change is covered in the -[other process](#for-all-other-documentation), so that time-sensitive documentation updates are -prioritized. - -### Documentation requirements - -Requirements for the documentation of a feature should be included as part of the -issue for planning that feature in a **Documentation** section within the issue description. Issues -created using the [**Feature Proposal** template](https://gitlab.com/gitlab-org/gitlab/raw/master/.gitlab/issue_templates/Feature%20proposal.md) -have this section by default. - -Anyone can add these details, but the Product Manager who assigns the issue to a specific release -milestone will ensure these details are present and finalized by the time of that milestone's kickoff. - -Developers, Technical Writers, and others may help further refine this plan at any time on request. - -The following details should be included: - -- What concepts and procedures should the documentation guide and enable the user to understand or - accomplish? -- To this end, what new page(s) are needed, if any? What pages or subsections need updates? - Consider changes and additions to user, admin, and API documentation. -- For any guide or instruction set, should it help address a single use case, or be flexible to - address a certain range of use cases? -- Do we need to update a previously recommended workflow? Should we link the new feature from - various relevant locations? Consider all ways documentation should be affected. -- Are there any key terms or task descriptions that should be included so that the documentation is - found in relevant searches? -- Include suggested titles of any pages or subsection headings, if applicable. -- List any documentation that should be cross-linked, if applicable. - -### Including docs with code - -Currently, the Technical Writing team strongly encourages including documentation in -the same merge request as the code that it relates to, but this is not strictly mandatory. -It's still common for documentation to be added in an MR separate from the feature MR. - -Engineering teams may elect to adopt a workflow where it is **mandatory** that docs -are included in the code MR, as part of their [definition of done](../contributing/merge_request_workflow.md#definition-of-done). -When a team adopts this workflow, that team's engineers must include their docs in the **same** -MR as their feature code, at all times. - -#### Downsides of separate docs MRs - -A workflow that has documentation separated into its own MR has many downsides. - -If the documentation merges **before** the feature: - -- GitLab.com users might try to use the feature before it's released, driving support tickets. -- If the feature is delayed, the documentation might not be pulled/reverted in time and could be - accidentally included in the self-managed package for that release. - -If the documentation merges **after** the feature: - -- The feature might be included in the self-managed package, but without any documentation - if the docs MR misses the cutoff. -- A feature might show up in the GitLab.com UI before any documentation exists for it. - Users surprised by this feature will search for documentation and won't find it, possibly driving - support tickets. - -Having two separate MRs means: - -- Two different people might be responsible for merging one feature, which is not workable - with an asynchronous work style. The feature might merge while the technical writer is asleep, - creating a potentially lengthy delay between the two merges. -- If the docs MR is assigned to the same maintainer as responsible for the feature - code MR, they will have to review and juggle two MRs instead of dealing with just one. - -Documentation quality might be lower, because: - -- Having docs in a separate MR will mean far fewer people will see and verify them, - increasing the likelihood that issues will be missed. -- In a "split" workflow, engineers might only create the documentation MR once the - feature MR is ready, or almost ready. This gives the technical writer little time - to learn about the feature in order to do a good review. It also increases pressure - on them to review and merge faster than desired, letting problems slip in due to haste. - -#### Benefits of always including docs with code - -Including docs with code (and doing it early in the development process) has many benefits: - -- There are no timing issues connected to releases: - - If a feature slips to the next release, the documentation slips too. - - If the feature *just* makes it into a release, the docs *just* make it in too. - - If a feature makes it to GitLab.com early, the documentation will be ready for - our early adopters. -- Only a single person will be responsible for merging the feature (the code maintainer). -- The technical writer will have more time to gain an understanding of the feature - and will be better able to verify the content of the docs in the Review App or GDK. - They will also be able to offer advice for improving the UI text or offer additional use cases. -- The documentation will have increased visibility: - - Everyone involved in the merge request will see the docs. This could include product - managers, multiple engineers with deep domain knowledge, as well as the code reviewers - and maintainer. They will be more likely to catch issues with examples, as well - as background or concepts that the technical writer may not be aware of. - - Increasing visibility of the documentation also has the side effect of improving - *other* engineers' documentation. By reviewing each other's MRs, each engineer's - own documentation skills will improve. -- Thinking about the documentation early can help engineers generate better examples, - as they will need to think about what examples a user will want, and will need to - make sure the code they write implements that example properly. - -#### Docs with code as a workflow - -In order to have docs included with code as a mandatory workflow, some changes might -need to happen to a team's current workflow: - -- The engineers must strive to include the docs early in the development process, - to give ample time for review, not just from the technical writer, but also the - code reviewer and maintainer. -- Reviewers and maintainers must also review the docs during code reviews, to make - sure the described processes match the expected use of the feature, and that examples - are correct. They do *not* need to worry about style, grammar, and so on. -- The technical writer must be assigned the MR directly and not only pinged. Thanks - to the ability to have [multiple assignees for any MR](../../user/project/merge_requests/getting_started.md#multiple-assignees-starter), - this can be done at any time, but must be before the code maintainer review. It's - common to have both the docs and code reviews happening at the same time, with the - author, reviewer and technical writer discussing the docs together. -- When the docs are ready, the technical writer will click **Approve** and usually - will no longer be involved in the MR. If the feature changes during code review and - the docs are updated, the technical writer must be reassigned the MR to verify the - update. -- Maintainers are allowed to merge features with the docs "as-is", even if the technical - writer has not given final approval yet. The **docs reviews must not be blockers**. Therefore - it's important to get the docs included and assigned to the technical writers early. - If the feature is merged before final docs approval, the maintainer must create - a [post-merge follow-up issue](#post-merge-reviews), and assign it to both the engineer - and technical writer. - -Maintainers are allowed to merge features with the docs "as-is" even if the -technical writer has not given final approval yet but the merge request has -all other required approvals. - -You can visualize the parallel workflow for code and docs reviews as: - -```mermaid -graph TD - A("Feature MR Created (Engineer)") --> |Assign| B("Code Review (reviewer)") - B --> |"Approve / Reassign"| C("Code Review (maintainer)") - C --> |Approve| F("Merge (maintainer)") - A --> D("Docs Added (Engineer)") - D --> |Assign| E("Docs Review (Tech Writer)") - E --> |Approve| F -``` - -For complex features split over multiple merge requests: - -- If a merge request is implementing components for a future feature, but the components - are not accessible to users yet, then no documentation should be included. -- If a merge request will expose a feature to users in any way, such as an enabled - UI element, an API endpoint, or anything similar, then that MR **must** have docs. - Note that this may mean multiple docs additions could happen in the buildup to the - implementation of a single large feature, for example API docs and feature usage docs. -- If it's unclear which engineer should add the feature documentation into their - MR, the engineering manager should decide during planning, and tie the documentation - to the last MR that must be merged before a feature is considered released. - This is often, but not always, a frontend MR. - -## For all other documentation - Documentation changes that are not associated with the release of a new or updated feature do not take the `~feature` label, but still need the `~documentation` label. They may include: - Documentation created or updated to improve accuracy, completeness, ease of use, or any reason - other than a [feature change](#for-a-product-change). + other than a [feature change](https://about.gitlab.com/handbook/engineering/ux/technical-writing/workflow/#for-a-product-change). - Addressing gaps in existing documentation, or making improvements to existing documentation. - Work on special projects related to the documentation. -TIP: **Tip:** -Anyone can contribute a merge request or create an issue for GitLab's documentation. - -### Who updates the docs - -Anyone can contribute! You can create a merge request for documentation when: - -- You find errors or other room for improvement in existing documentation. -- You have an idea for all-new documentation that would help a GitLab user or administrator to - accomplish their work with GitLab. - -### How to update the docs +## How to update the docs To update GitLab documentation: @@ -464,7 +110,7 @@ The process involves the following: The process is reflected in the **Documentation** [merge request template](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/merge_request_templates/Documentation.md). -### Other ways to help +## Other ways to help If you have ideas for further documentation resources please [create an issue](https://gitlab.com/gitlab-org/gitlab/issues/new?issuable_template=Documentation) diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index bd70d5b87ba..3951b0516e8 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -3,7 +3,8 @@ - **Write the code and the tests.**: As with any code, EE features should have good test coverage to prevent regressions. - **Write documentation.**: Add documentation to the `doc/` directory. Describe - the feature and include screenshots, if applicable. + the feature and include screenshots, if applicable. Indicate [what editions](documentation/styleguide.md#product-badges) + the feature applies to. - **Submit a MR to the `www-gitlab-com` project.**: Add the new feature to the [EE features list](https://about.gitlab.com/features/). @@ -111,7 +112,7 @@ There are a few gotchas with it: smaller methods. Or create a "hook" method that is empty in CE, and with the EE-specific implementation in EE. - when the original implementation contains a guard clause (e.g. - `return unless condition`), we cannot easily extend the behaviour by + `return unless condition`), we cannot easily extend the behavior by overriding the method, because we can't know when the overridden method (i.e. calling `super` in the overriding method) would want to stop early. In this case, we shouldn't just override it, but update the original method @@ -131,7 +132,7 @@ There are a few gotchas with it: ``` Instead of just overriding `Base#execute`, we should update it and extract - the behaviour into another method: + the behavior into another method: ```ruby class Base @@ -513,7 +514,7 @@ do that, so we'll follow regular object-oriented practices that we define the interface first here. For example, suppose we have a few more optional parameters for EE. We can move the -paramters out of the `Grape::API` class to a helper module, so we can inject it +parameters out of the `Grape::API` class to a helper module, so we can inject it before it would be used in the class. ```ruby @@ -613,9 +614,9 @@ module EE end ``` -#### EE-specific behaviour +#### EE-specific behavior -Sometimes we need EE-specific behaviour in some of the APIs. Normally we could +Sometimes we need EE-specific behavior in some of the APIs. Normally we could use EE methods to override CE methods, however API routes are not methods and therefore can't be simply overridden. We need to extract them into a standalone method, or introduce some "hooks" where we could inject behavior in the CE diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 758cecce315..185f536fc01 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -7,7 +7,7 @@ the [Elasticsearch integration documentation](../integration/elasticsearch.md#en ## Deep Dive -In June 2019, Mario de la Ossa hosted a [Deep Dive](https://gitlab.com/gitlab-org/create-stage/issues/1) on GitLab's [Elasticsearch integration](../integration/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction. +In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on GitLab's [Elasticsearch integration](../integration/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction. ## Supported Versions diff --git a/doc/development/emails.md b/doc/development/emails.md index a84895eef5b..e6e2ea8aae7 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -14,13 +14,11 @@ Please note that [S/MIME signed](../administration/smime_signing_email.md) email Rails provides a way to preview our mailer templates in HTML and plaintext using dummy data. -The previews live in [`app/mailers/previews`][previews] and can be viewed at +The previews live in [`app/mailers/previews`](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/mailers/previews) and can be viewed at [`/rails/mailers`](http://localhost:3000/rails/mailers). See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html#previewing-emails) for more information. -[previews]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/mailers/previews - ## Incoming email 1. Go to the GitLab installation directory. @@ -59,6 +57,9 @@ See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html# mailbox: "inbox" # The IDLE command timeout. idle_timeout: 60 + + # Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery + expunge_deleted: false ``` As mentioned, the part after `+` is ignored, and this will end up in the mailbox for `gitlab-incoming@gmail.com`. @@ -87,7 +88,7 @@ for the format of the email key: - Actions are always at the end, separated by `-`. For example `-issue` or `-merge-request` - If your feature is related to a project, the key begins with the project identifiers (project path slug - and project id), separated by `-`. For example, `gitlab-org-gitlab-foss-20` + and project ID), separated by `-`. For example, `gitlab-org-gitlab-foss-20` - Additional information, such as an author's token, can be added between the project identifiers and the action, separated by `-`. For example, `gitlab-org-gitlab-foss-20-Author_Token12345678-issue` - You register your handlers in `lib/gitlab/email/handler.rb` diff --git a/doc/development/event_tracking/backend.md b/doc/development/event_tracking/backend.md index dc4d7279671..93e135772ef 100644 --- a/doc/development/event_tracking/backend.md +++ b/doc/development/event_tracking/backend.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../telemetry/backend.md' +redirect_to: '../telemetry/index.md' --- -This document was moved to [another location](../../telemetry/backend.md). +This document was moved to [another location](../telemetry/index.md). diff --git a/doc/development/event_tracking/frontend.md b/doc/development/event_tracking/frontend.md index 0e98daf15bb..93e135772ef 100644 --- a/doc/development/event_tracking/frontend.md +++ b/doc/development/event_tracking/frontend.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../telemetry/frontend.md' +redirect_to: '../telemetry/index.md' --- -This document was moved to [another location](../../telemetry/frontend.md). +This document was moved to [another location](../telemetry/index.md). diff --git a/doc/development/event_tracking/index.md b/doc/development/event_tracking/index.md index ae555e99c6b..93e135772ef 100644 --- a/doc/development/event_tracking/index.md +++ b/doc/development/event_tracking/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../telemetry/index.md' +redirect_to: '../telemetry/index.md' --- -This document was moved to [another location](../../telemetry/index.md). +This document was moved to [another location](../telemetry/index.md). diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index ffa95d86876..f0e05139cba 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -32,10 +32,9 @@ The author then adds a comment to this piece of code and adds a link to the issu #... }, # Add your experiment here: - sign_up_flow: { - feature_toggle: :experimental_sign_up_flow, # Feature flag that will be used - environment: ::Gitlab.dev_env_or_com?, # Target environment - enabled_ratio: 0.1 # Percentage of users that will be part of the experiment. 10% of the users would be part of this experiments. + signup_flow: { + environment: ::Gitlab.dev_env_or_com?, # Target environment, defaults to enabled for development and GitLab.com + tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' # Used for providing the category when setting up tracking data } }.freeze ``` @@ -46,7 +45,7 @@ The author then adds a comment to this piece of code and adds a link to the issu class RegistrationController < Applicationcontroller def show # experiment_enabled?(:feature_name) is also available in views and helpers - if experiment_enabled?(:sign_up_flow) + if experiment_enabled?(:signup_flow) # render the experiment else # render the original version @@ -55,13 +54,18 @@ The author then adds a comment to this piece of code and adds a link to the issu end ``` -- Track necessary events. See the [telemetry guide](../../telemetry/index.md) for details. +- Track necessary events. See the [telemetry guide](../telemetry/index.md) for details. - After the merge request is merged, use [`chatops`](../../ci/chatops/README.md) in the -[appropriate channel](../feature_flags/controls.md#where-to-run-commands) to enable the feature flag and start the experiment. +[appropriate channel](../feature_flags/controls.md#communicate-the-change) to start the experiment for 10% of the users. +The feature flag should have the name of the experiment with the `_experiment_percentage` suffix appended. For visibility, please also share any commands run against production in the `#s_growth` channel: ```shell - /chatops run feature set --project=gitlab-org/gitlab experimental_sign_up_flow true + /chatops run feature set signup_flow_experiment_percentage 10 ``` - If you notice issues with the experiment, you can disable the experiment by setting the feature flag to `false` again. + If you notice issues with the experiment, you can disable the experiment by removing the feature flag: + + ```shell + /chatops run feature delete signup_flow_experiment_percentage + ``` diff --git a/doc/development/fe_guide/accessibility.md b/doc/development/fe_guide/accessibility.md index 4fd9a4fed60..998c71135fb 100644 --- a/doc/development/fe_guide/accessibility.md +++ b/doc/development/fe_guide/accessibility.md @@ -2,19 +2,12 @@ ## Resources -[Chrome Accessibility Developer Tools][chrome-accessibility-developer-tools] +[Chrome Accessibility Developer Tools](https://github.com/GoogleChrome/accessibility-developer-tools) are useful for testing for potential accessibility problems in GitLab. -The [axe][axe-website] browser extension (available for [Firefox][axe-firefox-extension] and [Chrome][axe-chrome-extension]) is +The [axe](https://www.deque.com/axe/) browser extension (available for [Firefox](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/) and [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd)) is also a handy tool for running audits and getting feedback on markup, CSS and even potentially problematic color usages. Accessibility best-practices and more in-depth information are available on -[the Audit Rules page][audit-rules] for the Chrome Accessibility Developer Tools. The "[awesome a11y][awesome-a11y]" list is also a +[the Audit Rules page](https://github.com/GoogleChrome/accessibility-developer-tools/wiki/Audit-Rules) for the Chrome Accessibility Developer Tools. The [Awesome Accessibility](https://github.com/brunopulis/awesome-a11y) list is also a useful compilation of accessibility-related material. - -[chrome-accessibility-developer-tools]: https://github.com/GoogleChrome/accessibility-developer-tools -[audit-rules]: https://github.com/GoogleChrome/accessibility-developer-tools/wiki/Audit-Rules -[axe-website]: https://www.deque.com/axe/ -[axe-firefox-extension]: https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/ -[axe-chrome-extension]: https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd -[awesome-a11y]: https://github.com/brunopulis/awesome-a11y diff --git a/doc/development/fe_guide/axios.md b/doc/development/fe_guide/axios.md index 6e7cf523f36..f8d301dac5e 100644 --- a/doc/development/fe_guide/axios.md +++ b/doc/development/fe_guide/axios.md @@ -1,13 +1,13 @@ # Axios -We use [axios][axios] to communicate with the server in Vue applications and most new code. +We use [axios](https://github.com/axios/axios) to communicate with the server in Vue applications and most new code. In order to guarantee all defaults are set you *should not use `axios` directly*, you should import `axios` from `axios_utils`. ## CSRF token All our request require a CSRF token. -To guarantee this token is set, we are importing [axios][axios], setting the token, and exporting `axios` . +To guarantee this token is set, we are importing [axios](https://github.com/axios/axios), setting the token, and exporting `axios` . This exported module should be used instead of directly using `axios` to ensure the token is set. @@ -32,19 +32,16 @@ This exported module should be used instead of directly using `axios` to ensure ## Mock axios response in tests -To help us mock the responses we are using [axios-mock-adapter][axios-mock-adapter]. +To help us mock the responses we are using [axios-mock-adapter](https://github.com/ctimmerm/axios-mock-adapter). -Advantages over [`spyOn()`]: +Advantages over [`spyOn()`](https://jasmine.github.io/api/edge/global.html#spyOn): - no need to create response objects - does not allow call through (which we want to avoid) - simple API to test error cases - provides `replyOnce()` to allow for different responses -We have also decided against using [axios interceptors] because they are not suitable for mocking. - -[axios interceptors]: https://github.com/axios/axios#interceptors -[`spyOn()`]: https://jasmine.github.io/api/edge/global.html#spyOn +We have also decided against using [axios interceptors](https://github.com/axios/axios#interceptors) because they are not suitable for mocking. ### Example @@ -77,8 +74,3 @@ Because polling function requires a header object, we need to always include an ```javascript mock.onGet('/users').reply(200, { foo: 'bar' }, {}); ``` - -[axios]: https://github.com/axios/axios -[axios-instance]: #creating-an-instance -[axios-interceptors]: https://github.com/axios/axios#interceptors -[axios-mock-adapter]: https://github.com/ctimmerm/axios-mock-adapter diff --git a/doc/development/fe_guide/design_patterns.md b/doc/development/fe_guide/design_patterns.md index 72a7861ffcb..5b7b16a9a8a 100644 --- a/doc/development/fe_guide/design_patterns.md +++ b/doc/development/fe_guide/design_patterns.md @@ -75,6 +75,4 @@ class Foo { new Foo({ container: '.my-element' }); ``` -You can find an example of the above in this [class][container-class-example]; - -[container-class-example]: https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/javascripts/mini_pipeline_graph_dropdown.js +You can find an example of the above in this [class](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/javascripts/mini_pipeline_graph_dropdown.js); diff --git a/doc/development/fe_guide/droplab/droplab.md b/doc/development/fe_guide/droplab/droplab.md index 4d7c882dc09..83bc4216403 100644 --- a/doc/development/fe_guide/droplab/droplab.md +++ b/doc/development/fe_guide/droplab/droplab.md @@ -26,7 +26,7 @@ If you do not provide any arguments, it will globally query and instantiate all <ul> ``` -```js +```javascript const droplab = new DropLab(); droplab.init(); ``` @@ -47,7 +47,7 @@ You can add static list items. <ul> ``` -```js +```javascript const droplab = new DropLab(); droplab.init(); ``` @@ -65,7 +65,7 @@ a non-global instance of DropLab using the `DropLab.prototype.init` method. <ul> ``` -```js +```javascript const trigger = document.getElementById('trigger'); const list = document.getElementById('list'); @@ -83,7 +83,7 @@ You can also add hooks to an existing DropLab instance using `DropLab.prototype. <ul id="list" data-dropdown><!-- ... --><ul> ``` -```js +```javascript const droplab = new DropLab(); droplab.init(); @@ -114,7 +114,7 @@ for all `data-dynamic` dropdown lists tracked by that DropLab instance. </ul> ``` -```js +```javascript const droplab = new DropLab(); droplab.init().addData([{ @@ -137,7 +137,7 @@ the data as the second argument and the `id` of the trigger element as the first </ul> ``` -```js +```javascript const droplab = new DropLab(); droplab.init().addData('trigger', [{ @@ -167,7 +167,7 @@ dropdown lists, one of which is dynamic. </div> ``` -```js +```javascript const droplab = new DropLab(); droplab.init().addData('trigger', [{ @@ -224,7 +224,7 @@ Some plugins require configuration values, the config object can be passed as th <ul id="list" data-dropdown><!-- ... --><ul> ``` -```js +```javascript const droplab = new DropLab(); const trigger = document.getElementById('trigger'); @@ -249,7 +249,7 @@ droplab.init(trigger, list, [droplabAjax], { When plugins are initialised for a droplab trigger+dropdown, DropLab will call the plugins `init` function, so this must be implemented in the plugin. -```js +```javascript class MyPlugin { static init() { this.someProp = 'someProp'; diff --git a/doc/development/fe_guide/droplab/plugins/ajax.md b/doc/development/fe_guide/droplab/plugins/ajax.md index 77ba6f739e6..abc208e7568 100644 --- a/doc/development/fe_guide/droplab/plugins/ajax.md +++ b/doc/development/fe_guide/droplab/plugins/ajax.md @@ -18,7 +18,7 @@ Add the `Ajax` object to the plugins array of a `DropLab.prototype.init` or `Dro <ul id="list" data-dropdown><!-- ... --><ul> ``` -```js +```javascript const droplab = new DropLab(); const trigger = document.getElementById('trigger'); diff --git a/doc/development/fe_guide/droplab/plugins/filter.md b/doc/development/fe_guide/droplab/plugins/filter.md index b867394a241..876149e4872 100644 --- a/doc/development/fe_guide/droplab/plugins/filter.md +++ b/doc/development/fe_guide/droplab/plugins/filter.md @@ -18,7 +18,7 @@ Add the `Filter` object to the plugins array of a `DropLab.prototype.init` or `D <ul> ``` -```js +```javascript const droplab = new DropLab(); const trigger = document.getElementById('trigger'); diff --git a/doc/development/fe_guide/droplab/plugins/input_setter.md b/doc/development/fe_guide/droplab/plugins/input_setter.md index db492da478a..9b2e1e8faab 100644 --- a/doc/development/fe_guide/droplab/plugins/input_setter.md +++ b/doc/development/fe_guide/droplab/plugins/input_setter.md @@ -23,7 +23,7 @@ You can also set the `InputSetter` config to an array of objects, which will all <ul> ``` -```js +```javascript const droplab = new DropLab(); const trigger = document.getElementById('trigger'); diff --git a/doc/development/fe_guide/event_tracking.md b/doc/development/fe_guide/event_tracking.md index ae555e99c6b..93e135772ef 100644 --- a/doc/development/fe_guide/event_tracking.md +++ b/doc/development/fe_guide/event_tracking.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../telemetry/index.md' +redirect_to: '../telemetry/index.md' --- -This document was moved to [another location](../../telemetry/index.md). +This document was moved to [another location](../telemetry/index.md). diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index 3b0b1d8f0da..8f8f162609a 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -39,7 +39,7 @@ bundle exec rake routes | grep "issues" ### 2. `modal_copy_button` vs `clipboard_button` -The `clipboard_button` uses the `copy_to_clipboard.js` behaviour, which is +The `clipboard_button` uses the `copy_to_clipboard.js` behavior, which is initialized on page load, so if there are vue-based clipboard buttons that don't exist at page load (such as ones in a `GlModal`), they do not have the click handlers associated with the clipboard package. diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 8ccc734ee35..caf84d04490 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -10,13 +10,13 @@ their execution by clicking **Execute query** button on the top left:  -We use [Apollo] and [Vue Apollo][vue-apollo] for working with GraphQL +We use [Apollo](https://www.apollographql.com/) and [Vue Apollo](https://github.com/vuejs/vue-apollo) for working with GraphQL on the frontend. ## Apollo Client To save duplicated clients getting created in different apps, we have a -[default client][default-client] that should be used. This setups the +[default client](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/javascripts/lib/graphql.js) that should be used. This setups the Apollo client with the correct URL and also sets the CSRF headers. Default client accepts two parameters: `resolvers` and `config`. @@ -73,7 +73,7 @@ More about fragments: ## Usage in Vue -To use Vue Apollo, import the [Vue Apollo][vue-apollo] plugin as well +To use Vue Apollo, import the [Vue Apollo](https://github.com/vuejs/vue-apollo) plugin as well as the default client. This should be created at the same point the Vue application is mounted. @@ -94,7 +94,7 @@ new Vue({ }); ``` -Read more about [Vue Apollo][vue-apollo] in the [Vue Apollo documentation](https://vue-apollo.netlify.com/guide/). +Read more about [Vue Apollo](https://github.com/vuejs/vue-apollo) in the [Vue Apollo documentation](https://vue-apollo.netlify.com/guide/). ### Local state with Apollo @@ -212,7 +212,7 @@ Read more about local state management with Apollo in the [Vue Apollo documentat When Apollo Client is used within Vuex and fetched data is stored in the Vuex store, there is no need in keeping Apollo Client cache enabled. Otherwise we would have data from the API stored in two places - Vuex store and Apollo Client cache. More to say, with Apollo default settings, a subsequent fetch from the GraphQL API could result in fetching data from Apollo cache (in the case where we have the same query and variables). To prevent this behavior, we need to disable Apollo Client cache passing a valid `fetchPolicy` option to its constructor: -```js +```javascript import fetchPolicies from '~/graphql_shared/fetch_policy_constants'; export const gqClient = createGqClient( @@ -298,7 +298,8 @@ handleClick() { GitLab's GraphQL API uses [Relay-style cursor pagination](https://www.apollographql.com/docs/react/data/pagination/#cursor-based) for connection types. This means a "cursor" is used to keep track of where in the data -set the next items should be fetched from. +set the next items should be fetched from. [GraphQL Ruby Connection Concepts](https://graphql-ruby.org/pagination/connection_concepts.html) +is a good overview and introduction to connections. Every connection type (for example, `DesignConnection` and `DiscussionConnection`) has a field `pageInfo` that contains an information required for pagination: @@ -426,7 +427,7 @@ This should be resolved in the scope of the issue #### Mocking response as component data -With [Vue test utils][vue-test-utils] it is easy to quickly test components that +With [Vue test utils](https://vue-test-utils.vuejs.org/) it is easy to quickly test components that fetch GraphQL queries. The simplest way is to use `shallowMount` and then set the data on the component @@ -598,11 +599,4 @@ defaultClient.query({ query }) .then(result => console.log(result)); ``` -Read more about the [Apollo] client in the [Apollo documentation](https://www.apollographql.com/docs/tutorial/client/). - -[Apollo]: https://www.apollographql.com/ -[vue-apollo]: https://github.com/vuejs/vue-apollo -[feature-flags]: ../feature_flags.md -[default-client]: https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/javascripts/lib/graphql.js -[vue-test-utils]: https://vue-test-utils.vuejs.org/ -[apollo-link-state]: https://www.apollographql.com/docs/link/links/state.html +Read more about the [Apollo](https://www.apollographql.com/) client in the [Apollo documentation](https://www.apollographql.com/docs/tutorial/client/). diff --git a/doc/development/fe_guide/icons.md b/doc/development/fe_guide/icons.md index ea321330c41..4fb738f5466 100644 --- a/doc/development/fe_guide/icons.md +++ b/doc/development/fe_guide/icons.md @@ -1,8 +1,8 @@ # Icons and SVG Illustrations -We manage our own Icon and Illustration library in the [`gitlab-svgs`][gitlab-svgs] repository. -This repository is published on [npm][npm] and managed as a dependency via yarn. -You can browse all available Icons and Illustrations [here][svg-preview]. +We manage our own Icon and Illustration library in the [`gitlab-svgs`](https://gitlab.com/gitlab-org/gitlab-svgs) repository. +This repository is published on [npm](https://www.npmjs.com/package/@gitlab/svgs) and managed as a dependency via yarn. +You can browse all available Icons and Illustrations [here](https://gitlab-org.gitlab.io/gitlab-svgs). To upgrade to a new version run `yarn upgrade @gitlab/svgs`. ## Icons @@ -22,7 +22,7 @@ sprite_icon(icon_name, size: nil, css_class: '') ``` - **icon_name** Use the icon_name that you can find in the SVG Sprite - ([Overview is available here][svg-preview]). + ([Overview is available here](https://gitlab-org.gitlab.io/gitlab-svgs)). - **size (optional)** Use one of the following sizes : 16, 24, 32, 48, 72 (this will be translated into a `s16` class) - **css_class (optional)** If you want to add additional css classes @@ -42,7 +42,7 @@ sprite_icon(icon_name, size: nil, css_class: '') ### Usage in Vue -[GitLab UI][gitlab-ui], our components library, provides a component to display sprite icons. +[GitLab UI](https://gitlab-org.gitlab.io/gitlab-ui/), our components library, provides a component to display sprite icons. Sample usage : @@ -65,7 +65,7 @@ export default { </template> ``` -- **name** Name of the Icon in the SVG Sprite ([Overview is available here][svg-preview]). +- **name** Name of the Icon in the SVG Sprite ([Overview is available here](https://gitlab-org.gitlab.io/gitlab-svgs)). - **size (optional)** Number value for the size which is then mapped to a specific CSS class (Available Sizes: 8, 12, 16, 18, 24, 32, 48, 72 are mapped to `sXX` css classes) - **css-classes (optional)** Additional CSS Classes to add to the svg tag. @@ -111,8 +111,3 @@ export default { <img :src="svgIllustrationPath" /> </template> ``` - -[npm]: https://www.npmjs.com/package/@gitlab/svgs -[gitlab-svgs]: https://gitlab.com/gitlab-org/gitlab-svgs -[svg-preview]: https://gitlab-org.gitlab.io/gitlab-svgs -[gitlab-ui]: https://gitlab-org.gitlab.io/gitlab-ui/ diff --git a/doc/development/fe_guide/index.md b/doc/development/fe_guide/index.md index 790cd94cec4..8fd6ba459b9 100644 --- a/doc/development/fe_guide/index.md +++ b/doc/development/fe_guide/index.md @@ -5,17 +5,17 @@ across GitLab's frontend team. ## Overview -GitLab is built on top of [Ruby on Rails](https://rubyonrails.org) using [Haml][haml] and also a JavaScript based Frontend with [Vue.js](https://vuejs.org). -Be wary of [the limitations that come with using Hamlit][hamlit-limits]. We also use [SCSS](https://sass-lang.com) and plain JavaScript with -modern ECMAScript standards supported through [Babel][babel] and ES module support through [webpack][webpack]. +GitLab is built on top of [Ruby on Rails](https://rubyonrails.org) using [Haml](http://haml.info/) and also a JavaScript based Frontend with [Vue.js](https://vuejs.org). +Be wary of [the limitations that come with using Hamlit](https://github.com/k0kubun/hamlit/blob/master/REFERENCE.md#limitations). We also use [SCSS](https://sass-lang.com) and plain JavaScript with +modern ECMAScript standards supported through [Babel](https://babeljs.io/) and ES module support through [webpack](https://webpack.js.org/). Working with our frontend assets requires Node (v10.13.0 or greater) and Yarn (v1.10.0 or greater). You can find information on how to install these on our -[installation guide][install]. +[installation guide](../../install/installation.md#4-node). ### Browser Support -For our currently-supported browsers, see our [requirements][requirements]. +For our currently-supported browsers, see our [requirements](../../install/requirements.md#supported-web-browsers). Use [BrowserStack](https://www.browserstack.com/) to test with our supported browsers. Login to BrowserStack with the credentials saved in GitLab's [shared 1Password account](https://about.gitlab.com/handbook/security/#1password-for-teams). @@ -83,7 +83,7 @@ Read the [frontend's FAQ](frontend_faq.md) for common small pieces of helpful in See the relevant style guides for our guidelines and for information on linting: - [JavaScript](style/javascript.md). Our guide is based on -the excellent [Airbnb][airbnb-js-style-guide] style guide with a few small +the excellent [Airbnb](https://github.com/airbnb/javascript) style guide with a few small changes. - [SCSS](style/scss.md): our SCSS conventions which are enforced through [`scss-lint`](https://github.com/sds/scss-lint). - [HTML](style/html.md). Guidelines for writing HTML code consistent with the rest of the codebase. @@ -109,14 +109,3 @@ Our accessibility standards and resources. Frontend internationalization support is described in [this document](../i18n/). The [externalization part of the guide](../i18n/externalization.md) explains the helpers/methods available. - -[haml]: http://haml.info/ -[hamlit]: https://github.com/k0kubun/hamlit -[hamlit-limits]: https://github.com/k0kubun/hamlit/blob/master/REFERENCE.md#limitations -[babel]: https://babeljs.io/ -[webpack]: https://webpack.js.org/ -[jquery]: https://jquery.com/ -[axios]: https://github.com/axios/axios -[airbnb-js-style-guide]: https://github.com/airbnb/javascript -[install]: ../../install/installation.md#4-node -[requirements]: ../../install/requirements.md#supported-web-browsers diff --git a/doc/development/fe_guide/performance.md b/doc/development/fe_guide/performance.md index fcba8758c2f..aaa6bb16fab 100644 --- a/doc/development/fe_guide/performance.md +++ b/doc/development/fe_guide/performance.md @@ -41,9 +41,9 @@ But in general it should be handled automatically through a `MutationObserver` i Only animate `opacity` & `transform` properties. Other properties (such as `top`, `left`, `margin`, and `padding`) all cause Layout to be recalculated, which is much more expensive. For details on this, see "Styles that Affect Layout" in -[High Performance Animations][high-perf-animations]. +[High Performance Animations](https://www.html5rocks.com/en/tutorials/speed/high-performance-animations/). -If you _do_ need to change layout (e.g. a sidebar that pushes main content over), prefer [FLIP][flip] to change expensive +If you _do_ need to change layout (e.g. a sidebar that pushes main content over), prefer [FLIP](https://aerotwist.com/blog/flip-your-animations/) to change expensive properties once, and handle the actual animation with transforms. ## Reducing Asset Footprint @@ -160,18 +160,13 @@ General tips: - If some functionality can reasonably be achieved without adding extra libraries, avoid them. - Use page-specific JavaScript as described above to load libraries that are only needed on certain pages. - Use code-splitting dynamic imports wherever possible to lazy-load code that is not needed initially. -- [High Performance Animations][high-perf-animations] +- [High Performance Animations](https://www.html5rocks.com/en/tutorials/speed/high-performance-animations/) --- ## Additional Resources - [WebPage Test](https://www.webpagetest.org) for testing site loading time and size. -- [Google PageSpeed Insights][pagespeed-insights] grades web pages and provides feedback to improve the page. +- [Google PageSpeed Insights](https://developers.google.com/speed/pagespeed/insights/) grades web pages and provides feedback to improve the page. - [Profiling with Chrome DevTools](https://developers.google.com/web/tools/chrome-devtools/) -- [Browser Diet][browser-diet] is a community-built guide that catalogues practical tips for improving web page performance. - -[pagespeed-insights]: https://developers.google.com/speed/pagespeed/insights/ -[browser-diet]: https://browserdiet.com/ -[high-perf-animations]: https://www.html5rocks.com/en/tutorials/speed/high-performance-animations/ -[flip]: https://aerotwist.com/blog/flip-your-animations/ +- [Browser Diet](https://browserdiet.com/) is a community-built guide that catalogues practical tips for improving web page performance. diff --git a/doc/development/fe_guide/security.md b/doc/development/fe_guide/security.md index 7dba61d6b45..a001dd83ab7 100644 --- a/doc/development/fe_guide/security.md +++ b/doc/development/fe_guide/security.md @@ -2,8 +2,8 @@ ## Resources -[Mozilla’s HTTP Observatory CLI][observatory-cli] and the -[Qualys SSL Labs Server Test][qualys-ssl] are good resources for finding +[Mozilla’s HTTP Observatory CLI](https://github.com/mozilla/http-observatory-cli) and the +[Qualys SSL Labs Server Test](https://www.ssllabs.com/ssltest/analyze.html) are good resources for finding potential problems and ensuring compliance with security best practices. <!-- Uncomment these sections when CSP/SRI are implemented. @@ -29,14 +29,14 @@ Some exceptions include: - Connecting with GitHub, Bitbucket, GitLab.com, etc. to allow project importing. - Connecting with Google, Twitter, GitHub, etc. to allow OAuth authentication. -We use [the Secure Headers gem][secure_headers] to enable Content +We use [the Secure Headers gem](https://github.com/twitter/secureheaders) to enable Content Security Policy headers in the GitLab Rails app. Some resources on implementing Content Security Policy: -- [MDN Article on CSP][mdn-csp] -- [GitHub’s CSP Journey on the GitHub Engineering Blog][github-eng-csp] -- The Dropbox Engineering Blog's series on CSP: [1][dropbox-csp-1], [2][dropbox-csp-2], [3][dropbox-csp-3], [4][dropbox-csp-4] +- [MDN Article on CSP](https://developer.mozilla.org/en-US/docs/Web/Security/CSP) +- [GitHub’s CSP Journey on the GitHub Engineering Blog](http://githubengineering.com/githubs-csp-journey/) +- The Dropbox Engineering Blog's series on CSP: [1](https://blogs.dropbox.com/tech/2015/09/on-csp-reporting-and-filtering/), [2](https://blogs.dropbox.com/tech/2015/09/unsafe-inline-and-nonce-deployment/), [3](https://blogs.dropbox.com/tech/2015/09/csp-the-unexpected-eval/), [4](https://blogs.dropbox.com/tech/2015/09/csp-third-party-integrations-and-privilege-separation/) ### Subresource Integrity (SRI) @@ -52,8 +52,8 @@ All CSS and JavaScript assets should use Subresource Integrity. Some resources on implementing Subresource Integrity: -- [MDN Article on SRI][mdn-sri] -- [Subresource Integrity on the GitHub Engineering Blog][github-eng-sri] +- [MDN Article on SRI](https://developer.mozilla.org/en-us/docs/web/security/subresource_integrity) +- [Subresource Integrity on the GitHub Engineering Blog](http://githubengineering.com/subresource-integrity/) --> @@ -67,7 +67,7 @@ such as with reCAPTCHA, which cannot be used without an `iframe`. ## Avoiding inline scripts and styles -In order to protect users from [XSS vulnerabilities][xss], we will disable +In order to protect users from [XSS vulnerabilities](https://en.wikipedia.org/wiki/Cross-site_scripting), we will disable inline scripts in the future using Content Security Policy. While inline scripts can be useful, they're also a security concern. If @@ -77,16 +77,3 @@ inject scripts into the web app. Inline styles should be avoided in almost all cases, they should only be used when no alternatives can be found. This allows reusability of styles as well as readability. - -[observatory-cli]: https://github.com/mozilla/http-observatory-cli -[qualys-ssl]: https://www.ssllabs.com/ssltest/analyze.html -[secure_headers]: https://github.com/twitter/secureheaders -[mdn-csp]: https://developer.mozilla.org/en-US/docs/Web/Security/CSP -[github-eng-csp]: http://githubengineering.com/githubs-csp-journey/ -[dropbox-csp-1]: https://blogs.dropbox.com/tech/2015/09/on-csp-reporting-and-filtering/ -[dropbox-csp-2]: https://blogs.dropbox.com/tech/2015/09/unsafe-inline-and-nonce-deployment/ -[dropbox-csp-3]: https://blogs.dropbox.com/tech/2015/09/csp-the-unexpected-eval/ -[dropbox-csp-4]: https://blogs.dropbox.com/tech/2015/09/csp-third-party-integrations-and-privilege-separation/ -[mdn-sri]: https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity -[github-eng-sri]: http://githubengineering.com/subresource-integrity/ -[xss]: https://en.wikipedia.org/wiki/Cross-site_scripting diff --git a/doc/development/fe_guide/style/javascript.md b/doc/development/fe_guide/style/javascript.md index 7951c702601..b69a6f1941c 100644 --- a/doc/development/fe_guide/style/javascript.md +++ b/doc/development/fe_guide/style/javascript.md @@ -184,6 +184,9 @@ This can help to quickly understand the control flow. // bad if (isThingNull) return ''; +if (isThingNull) + return ''; + // good if (isThingNull) { return ''; @@ -192,7 +195,7 @@ if (isThingNull) { ## ESLint -ESLint behaviour can be found in our [tooling guide](../tooling.md). +ESLint behavior can be found in our [tooling guide](../tooling.md). ## IIFEs diff --git a/doc/development/fe_guide/style/scss.md b/doc/development/fe_guide/style/scss.md index 6c0247ad00c..336c9b8ca35 100644 --- a/doc/development/fe_guide/style/scss.md +++ b/doc/development/fe_guide/style/scss.md @@ -9,30 +9,25 @@ easy to maintain, and performant for the end-user. ## Rules +Our CSS is a mixture of current and legacy approaches. That means sometimes it may be difficult to follow this guide to the letter; it means you will definitely run into exceptions, where following the guide is difficult to impossible without outsized effort. In those cases, you may work with your reviewers and maintainers to identify an approach that does not fit these rules. Please endeavor to limit these cases. + ### Utility Classes -As part of the effort for [cleaning up our CSS and moving our components into `gitlab-ui`](https://gitlab.com/groups/gitlab-org/-/epics/950) -led by the [GitLab UI WG](https://gitlab.com/gitlab-com/www-gitlab-com/-/merge_requests/20623) we prefer the use of utility classes over adding new CSS. However, complex CSS can be addressed by adding component classes. +In order to reduce the generation of more CSS as our site grows, prefer the use of utility classes over adding new CSS. In complex cases, CSS can be addressed by adding component classes. #### Where are utility classes defined? -- [Bootstrap's Utility Classes](https://getbootstrap.com/docs/4.3/utilities/) -- [`common.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/stylesheets/framework/common.scss) (old) -- [`utilities.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/stylesheets/utilities.scss) (new) +Prefer the use of [utility classes defined in GitLab UI](https://gitlab.com/gitlab-org/gitlab-ui/-/blob/master/doc/css.md#utilities). An easy list of classes can also be [seen on Unpkg](https://unpkg.com/browse/@gitlab/ui/src/scss/utilities.scss). -#### Where should I put new utility classes? +Classes in [`utilities.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/stylesheets/utilities.scss) and [`common.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/stylesheets/framework/common.scss) are being deprecated. Classes in [`common.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/stylesheets/framework/common.scss) that use non-design system values should be avoided in favor of conformant values. -New utility classes should be added to [`utilities.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/stylesheets/utilities.scss). Existing classes include: +Avoid [Bootstrap's Utility Classes](https://getbootstrap.com/docs/4.3/utilities/). -| Name | Pattern | Example | -|------|---------|---------| -| Background color | `.bg-{variant}-{shade}` | `.bg-warning-400` | -| Text color | `.text-{variant}-{shade}` | `.text-success-500` | -| Font size | `.text-{size}` | `.text-2` | +#### Where should I put new utility classes? -- `{variant}` is one of 'primary', 'secondary', 'success', 'warning', 'error' -- `{shade}` is one of the shades listed on [colors](https://design.gitlab.com/product-foundations/colors/) -- `{size}` is a number from 1-6 from our [Type scale](https://design.gitlab.com/product-foundations/typography/) +If a class you need has not been added to GitLab UI, you get to add it! Follow the naming patterns documented in the [utility files](https://gitlab.com/gitlab-org/gitlab-ui/-/tree/master/src/scss/utility-mixins) and refer to [GitLab UI's CSS documentation](https://gitlab.com/gitlab-org/gitlab-ui/-/blob/master/doc/contributing/adding_css.md#adding-utility-mixins) for more details, especially about adding responsive and stateful rules. + +If it is not possible to wait for a GitLab UI update (generally one day), add the class to [`utilities.scss`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/stylesheets/utilities.scss) following the same naming conventions documented in GitLab UI. A follow—up issue to backport the class to GitLab UI and delete it from GitLab should be opened. #### When should I create component classes? @@ -77,6 +72,24 @@ CSS classes should use the `lowercase-hyphenated` format rather than } ``` +Class names should be used instead of tag name selectors. +Using tag name selectors are discouraged in CSS because +they can affect unintended elements in the hierarchy. +Also, since they are not meaningful names, they do not +add meaning to the code. + +```scss +// Bad +ul { + color: #fff; +} + +// Good +.class-name { + color: #fff; +} +``` + ### Formatting You should always use a space before a brace, braces should be on the same @@ -254,8 +267,8 @@ documentation includes [a full list of their linters](https://github.com/sds/scs ### Fixing issues If you want to automate changing a large portion of the codebase to conform to -the SCSS style guide, you can use [CSSComb][csscomb]. First install -[Node][node] and [NPM][npm], then run `npm install csscomb -g` to install +the SCSS style guide, you can use [CSSComb](https://github.com/csscomb/csscomb.js). First install +[Node](https://github.com/nodejs/node) and [NPM](https://www.npmjs.com/), then run `npm install csscomb -g` to install CSSComb globally (system-wide). Run it in the GitLab directory with `csscomb app/assets/stylesheets` to automatically fix issues with CSS/SCSS. @@ -279,7 +292,3 @@ Make sure a comment is added on the line above the `disable` rule, otherwise the linter will throw a warning. `DisableLinterReason` is enabled to make sure the style guide isn't being ignored, and to communicate to others why the style guide is ignored in this instance. - -[csscomb]: https://github.com/csscomb/csscomb.js -[node]: https://github.com/nodejs/node -[npm]: https://www.npmjs.com/ diff --git a/doc/development/fe_guide/style/vue.md b/doc/development/fe_guide/style/vue.md index a7d8fc61752..46305cc7217 100644 --- a/doc/development/fe_guide/style/vue.md +++ b/doc/development/fe_guide/style/vue.md @@ -53,7 +53,7 @@ Please check this [rules](https://github.com/vuejs/eslint-plugin-vue#bulb-rules) ## Naming -1. **Extensions**: Use `.vue` extension for Vue components. Do not use `.js` as file extension ([#34371]). +1. **Extensions**: Use `.vue` extension for Vue components. Do not use `.js` as file extension ([#34371](https://gitlab.com/gitlab-org/gitlab-foss/issues/34371)). 1. **Reference Naming**: Use PascalCase for their instances: ```javascript @@ -89,8 +89,6 @@ Please check this [rules](https://github.com/vuejs/eslint-plugin-vue#bulb-rules) <component my-prop="prop" /> ``` -[#34371]: https://gitlab.com/gitlab-org/gitlab-foss/issues/34371 - ## Alignment 1. Follow these alignment styles for the template method: @@ -327,7 +325,7 @@ When using `v-for` you need to provide a *unique* `:key` attribute for each item </div> ``` -1. When the elements being iterated don't have a unique id, you can use the array index as the `:key` attribute +1. When the elements being iterated don't have a unique ID, you can use the array index as the `:key` attribute ```html <div diff --git a/doc/development/fe_guide/tooling.md b/doc/development/fe_guide/tooling.md index 0e8f5ed05ed..585cd969c96 100644 --- a/doc/development/fe_guide/tooling.md +++ b/doc/development/fe_guide/tooling.md @@ -56,13 +56,13 @@ When declaring multiple globals, always use one `/* global [name] */` line per v ## Formatting with Prettier -Our code is automatically formatted with [Prettier](https://prettier.io) to follow our style guides. Prettier is taking care of formatting .js, .vue, and .scss files based on the standard prettier rules. You can find all settings for Prettier in `.prettierrc`. +Our code is automatically formatted with [Prettier](https://prettier.io) to follow our style guides. Prettier is taking care of formatting `.js`, `.vue`, and `.scss` files based on the standard prettier rules. You can find all settings for Prettier in `.prettierrc`. ### Editor The easiest way to include prettier in your workflow is by setting up your preferred editor (all major editors are supported) accordingly. We suggest setting up prettier to run automatically when each file is saved. Find [here](https://prettier.io/docs/en/editors.html) the best way to set it up in your preferred editor. -Please take care that you only let Prettier format the same file types as the global Yarn script does (.js, .vue, and .scss). In VSCode by example you can easily exclude file formats in your settings file: +Please take care that you only let Prettier format the same file types as the global Yarn script does (`.js`, `.vue`, and `.scss`). In VSCode by example you can easily exclude file formats in your settings file: ```json "prettier.disableLanguages": [ diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index aeedd57fd83..972c2ded9c9 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -12,17 +12,17 @@ What is described in the following sections can be found in these examples: ## Vue architecture -All new features built with Vue.js must follow a [Flux architecture][flux]. +All new features built with Vue.js must follow a [Flux architecture](https://facebook.github.io/flux/). The main goal we are trying to achieve is to have only one data flow and only one data entry. In order to achieve this goal we use [vuex](#vuex). -You can also read about this architecture in vue docs about [state management][state-management] -and about [one way data flow][one-way-data-flow]. +You can also read about this architecture in vue docs about [state management](https://vuejs.org/v2/guide/state-management.html#Simple-State-Management-from-Scratch) +and about [one way data flow](https://vuejs.org/v2/guide/components.html#One-Way-Data-Flow). ### Components and Store -In some features implemented with Vue.js, like the [issue board][issue-boards] -or [environments table][environments-table] +In some features implemented with Vue.js, like the [issue board](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/boards) +or [environments table](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/environments) you can find a clear separation of concerns: ```plaintext @@ -47,7 +47,7 @@ of the new feature should be. The Store and the Service should be imported and initialized in this file and provided as a prop to the main component. -Be sure to read about [page-specific JavaScript][page_specific_javascript]. +Be sure to read about [page-specific JavaScript](./performance.md#page-specific-javascript). ### Bootstrapping Gotchas @@ -162,7 +162,7 @@ For example, tables are used in a quite amount of places across GitLab, a table would be a good fit for a component. On the other hand, a table cell used only in one table would not be a good use of this pattern. -You can read more about components in Vue.js site, [Component System][component-system] +You can read more about components in Vue.js site, [Component System](https://vuejs.org/v2/guide/#Composing-with-Components). ### A folder for the Store @@ -189,96 +189,135 @@ Each Vue component has a unique output. This output is always present in the ren Although we can test each method of a Vue component individually, our goal must be to test the output of the render/template function, which represents the state at all times. -Make use of the [axios mock adapter](axios.md#mock-axios-response-in-tests) to mock data returned. - -Here's how we would test the Todo App above: +Here's an example of a well structured unit test for [this Vue component](#appendix---vue-component-subject-under-test): ```javascript -import Vue from 'vue'; -import axios from '~/lib/utils/axios_utils'; +import { shallowMount } from '@vue/test-utils'; +import { GlLoadingIcon } from '@gitlab/ui'; import MockAdapter from 'axios-mock-adapter'; +import axios from '~/lib/utils/axios_utils'; +import App from '~/todos/app.vue'; + +const TEST_TODOS = [ + { text: 'Lorem ipsum test text' }, + { text: 'Lorem ipsum 2' }, +]; +const TEST_NEW_TODO = 'New todo title'; +const TEST_TODO_PATH = '/todos'; -describe('Todos App', () => { - let vm; +describe('~/todos/app.vue', () => { + let wrapper; let mock; beforeEach(() => { - // Create a mock adapter for stubbing axios API requests + // IMPORTANT: Use axios-mock-adapter for stubbing axios API requests mock = new MockAdapter(axios); - - const Component = Vue.extend(component); - - // Mount the Component - vm = new Component().$mount(); + mock.onGet(TEST_TODO_PATH).reply(200, TEST_TODOS); + mock.onPost(TEST_TODO_PATH).reply(200); }); afterEach(() => { - // Reset the mock adapter - mock.restore(); - // Destroy the mounted component - vm.$destroy(); - }); + // IMPORTANT: Clean up the component instance and axios mock adapter + wrapper.destroy(); + wrapper = null; - it('should render the loading state while the request is being made', () => { - expect(vm.$el.querySelector('i.fa-spin')).toBeDefined(); + mock.restore(); }); - it('should render todos returned by the endpoint', done => { - // Mock the get request on the API endpoint to return data - mock.onGet('/todos').replyOnce(200, [ - { - title: 'This is a todo', - text: 'This is the text', + // NOTE: It is very helpful to separate setting up the component from + // its collaborators (i.e. Vuex, axios, etc.) + const createWrapper = (props = {}) => { + wrapper = shallowMount(App, { + propsData: { + path: TEST_TODO_PATH, + ...props, }, - ]); + }); + }; + // NOTE: Helper methods greatly help test maintainability and readability. + const findLoader = () => wrapper.find(GlLoadingIcon); + const findAddButton = () => wrapper.find('[data-testid="add-button"]'); + const findTextInput = () => wrapper.find('[data-testid="text-input"]'); + const findTodoData = () => wrapper.findAll('[data-testid="todo-item"]').wrappers.map(wrapper => ({ text: wrapper.text() })); + + describe('when mounted and loading', () => { + beforeEach(() => { + // Create request which will never resolve + mock.onGet(TEST_TODO_PATH).reply(() => new Promise(() => {})); + createWrapper(); + }); - Vue.nextTick(() => { - const items = vm.$el.querySelectorAll('.js-todo-list div') - expect(items.length).toBe(1); - expect(items[0].textContent).toContain('This is the text'); - done(); + it('should render the loading state', () => { + expect(findLoader().exists()).toBe(true); }); }); - it('should add a todos on button click', (done) => { + describe('when todos are loaded', () => { + beforeEach(() => { + createWrapper(); + // IMPORTANT: This component fetches data asynchronously on mount, so let's wait for the Vue template to update + return wrapper.vm.$nextTick(); + }); - // Mock the put request and check that the sent data object is correct - mock.onPut('/todos').replyOnce((req) => { - expect(req.data).toContain('text'); - expect(req.data).toContain('title'); + it('should not show loading', () => { + expect(findLoader().exists()).toBe(false); + }); - return [201, {}]; + it('should render todos', () => { + expect(findTodoData()).toEqual(TEST_TODOS); }); - vm.$el.querySelector('.js-add-todo').click(); + it('when todo is added, should post new todo', () => { + findTextInput().vm.$emit('update', TEST_NEW_TODO) + findAddButton().vm.$emit('click'); - // Add a new interceptor to mock the add Todo request - Vue.nextTick(() => { - expect(vm.$el.querySelectorAll('.js-todo-list div').length).toBe(2); - done(); + return wrapper.vm.$nextTick() + .then(() => { + expect(mock.history.post.map(x => JSON.parse(x.data))).toEqual([{ text: TEST_NEW_TODO }]); + }); }); }); }); ``` -### `mountComponent` helper +### Test the component's output -There is a helper in `spec/javascripts/helpers/vue_mount_component_helper.js` that allows you to mount a component with the given props: +The main return value of a Vue component is the rendered output. In order to test the component we +need to test the rendered output. [Vue](https://vuejs.org/v2/guide/unit-testing.html) guide's to unit test show us exactly that: + +### Events + +We should test for events emitted in response to an action within our component, this is useful to verify the correct events are being fired with the correct arguments. + +For any DOM events we should use [`trigger`](https://vue-test-utils.vuejs.org/api/wrapper/#trigger) to fire out event. ```javascript -import Vue from 'vue'; -import mountComponent from 'spec/helpers/vue_mount_component_helper' -import component from 'component.vue' +// Assuming SomeButton renders: <button>Some button</button> +wrapper = mount(SomeButton); -const Component = Vue.extend(component); -const data = {prop: 'foo'}; -const vm = mountComponent(Component, data); +... +it('should fire the click event', () => { + const btn = wrapper.find('button') + + btn.trigger('click'); + ... +}) ``` -### Test the component's output +When we need to fire a Vue event, we should use [`emit`](https://vuejs.org/v2/guide/components-custom-events.html) to fire our event. -The main return value of a Vue component is the rendered output. In order to test the component we -need to test the rendered output. [Vue][vue-test] guide's to unit test show us exactly that: +```javascript +wrapper = shallowMount(DropdownItem); + +... + +it('should fire the itemClicked event', () => { + DropdownItem.vm.$emit('itemClicked'); + ... +}) +``` + +We should verify an event has been fired by asserting against the result of the [`emitted()`](https://vue-test-utils.vuejs.org/api/wrapper/#emitted) method ## Vue.js Expert Role @@ -287,15 +326,50 @@ One should apply to be a Vue.js expert by opening an MR when the Merge Request's - Deep understanding of Vue and Vuex reactivity - Vue and Vuex code are structured according to both official and our guidelines - Full understanding of testing a Vue and Vuex application -- Vuex code follows the [documented pattern](vuex.md#actions-pattern-request-and-receive-namespaces) +- Vuex code follows the [documented pattern](vuex.md#naming-pattern-request-and-receive-namespaces) - Knowledge about the existing Vue and Vuex applications and existing reusable components -[issue-boards]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/boards -[environments-table]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/environments -[page_specific_javascript]: ./performance.md#page-specific-javascript -[component-system]: https://vuejs.org/v2/guide/#Composing-with-Components -[state-management]: https://vuejs.org/v2/guide/state-management.html#Simple-State-Management-from-Scratch -[one-way-data-flow]: https://vuejs.org/v2/guide/components.html#One-Way-Data-Flow -[vue-test]: https://vuejs.org/v2/guide/unit-testing.html -[flux]: https://facebook.github.io/flux/ -[axios]: https://github.com/axios/axios +## Vue 2 -> Vue 3 Migration + +> This section is added temporarily to support the efforts to migrate the codebase from Vue 2.x to Vue 3.x + +Currently, we recommend to minimize adding certain features to the codebase to prevent increasing the tech debt for the eventual migration: + +- filters; +- event buses; +- functional templated +- `slot` attributes + +You can find more details on [Migration to Vue 3](vue3_migration.md) + +## Appendix - Vue component subject under test + +This is the template for the example component which is tested in the [Testing Vue components](#testing-vue-components) section: + +```html +<template> + <div class="content"> + <gl-loading-icon v-if="isLoading" /> + <template v-else> + <div + v-for="todo in todos" + :key="todo.id" + :class="{ 'gl-strike': todo.isDone }" + data-testid="todo-item" + >{{ toddo.text }}</div> + <footer class="gl-border-t-1 gl-mt-3 gl-pt-3"> + <gl-form-input + type="text" + v-model="todoText" + data-testid="text-input" + > + <gl-button + variant="success" + data-testid="add-button" + @click="addTodo" + >Add</gl-button> + </footer> + </template> + </div> +</template> +``` diff --git a/doc/development/fe_guide/vue3_migration.md b/doc/development/fe_guide/vue3_migration.md new file mode 100644 index 00000000000..7ab48db7f76 --- /dev/null +++ b/doc/development/fe_guide/vue3_migration.md @@ -0,0 +1,124 @@ +# Migration to Vue 3 + +In order to prepare for the eventual migration to Vue 3.x, we should be wary about adding the following features to the codebase: + +## Vue filters + +**Why?** + +Filters [are removed](https://github.com/vuejs/rfcs/blob/master/active-rfcs/0015-remove-filters.md) from the Vue 3 API completely. + +**What to use instead** + +Component's computed properties / methods or external helpers. + +## Event hub + +**Why?** + +`$on`, `$once`, and `$off` methods [are removed](https://github.com/vuejs/rfcs/blob/master/active-rfcs/0020-events-api-change.md) from the Vue instance, so in Vue 3 it can't be used to create an event hub. + +**What to use instead** + +Vue docs recommend using [mitt](https://github.com/developit/mitt) library. It's relatively small (200 bytes gzipped) and has a simple API: + +```javascript +import mitt from 'mitt' + +const emitter = mitt() + +// listen to an event +emitter.on('foo', e => console.log('foo', e) ) + +// listen to all events +emitter.on('*', (type, e) => console.log(type, e) ) + +// fire an event +emitter.emit('foo', { a: 'b' }) + +// working with handler references: +function onFoo() {} + +emitter.on('foo', onFoo) // listen +emitter.off('foo', onFoo) // unlisten +``` + +**Event hub factory** + +To make it easier for you to migrate existing event hubs to the new recommended approach, or simply +to create new ones, we have created a factory that you can use to instantiate a new mitt-based +event hub. + +```javascript +import createEventHub from '~/helpers/event_hub_factory'; + +export default createEventHub(); +``` + +Event hubs created with the factory expose the same methods as Vue 2 event hubs (`$on`, `$once`, `$off` and +`$emit`), making them backward compatible with our previous approach. + +## \<template functional> + +**Why?** + +In Vue 3, `{ functional: true }` option [is removed](https://github.com/vuejs/rfcs/blob/functional-async-api-change/active-rfcs/0007-functional-async-api-change.md) and `<template functional>` is no longer supported. + +**What to use instead** + +Functional components must be written as plain functions: + +```javascript +import { h } from 'vue' + +const FunctionalComp = (props, slots) => { + return h('div', `Hello! ${props.name}`) +} +``` + +## Old slots syntax with `slot` attribute + +**Why?** + +In Vue 2.6 `slot` attribute was already deprecated in favor of `v-slot` directive but its usage is still allowed and sometimes we prefer using them because it simplifies unit tests (with old syntax, slots are rendered on `shallowMount`). However, in Vue 3 we can't use old syntax anymore. + +**What to use instead** + +The syntax with `v-slot` directive. To fix rendering slots in `shallowMount`, we need to stub a child component with slots explicitly. + +```html +<!-- MyAwesomeComponent.vue --> +<script> +import SomeChildComponent from './some_child_component.vue' + +export default { + components: { + SomeChildComponent + } +} + +</script> + +<template> + <div> + <h1>Hello GitLab!</h1> + <some-child-component> + <template #header> + Header content + </template> + </some-child-component> + </div> +</template> +``` + +```javascript +// MyAwesomeComponent.spec.js + +import SomeChildComponent from '~/some_child_component.vue' + +shallowMount(MyAwesomeComponent, { + stubs: { + SomeChildComponent + } +}) +``` diff --git a/doc/development/fe_guide/vuex.md b/doc/development/fe_guide/vuex.md index 65661b0cc3b..e7be67b8da5 100644 --- a/doc/development/fe_guide/vuex.md +++ b/doc/development/fe_guide/vuex.md @@ -1,6 +1,6 @@ # Vuex -When there's a clear benefit to separating state management from components (e.g. due to state complexity) we recommend using [Vuex][vuex-docs] over any other Flux pattern. Otherwise, feel free to manage state within the components. +When there's a clear benefit to separating state management from components (e.g. due to state complexity) we recommend using [Vuex](https://vuex.vuejs.org) over any other Flux pattern. Otherwise, feel free to manage state within the components. Vuex should be strongly considered when: @@ -9,7 +9,7 @@ Vuex should be strongly considered when: - There are complex interactions with Backend, e.g. multiple API calls - The app involves interacting with backend via both traditional REST API and GraphQL (especially when moving the REST API over to GraphQL is a pending backend task) -_Note:_ All of the below is explained in more detail in the official [Vuex documentation][vuex-docs]. +_Note:_ All of the below is explained in more detail in the official [Vuex documentation](https://vuex.vuejs.org). ## Separation of concerns @@ -86,71 +86,35 @@ You can use `mapState` to access state properties in the components. An action is a payload of information to send data from our application to our store. -An action is usually composed by a `type` and a `payload` and they describe what happened. -Enforcing that every change is described as an action lets us have a clear understanding of what is going on in the app. +An action is usually composed by a `type` and a `payload` and they describe what happened. Unlike [mutations](#mutationsjs), actions can contain asynchronous operations - that's why we always need to handle asynchronous logic in actions. -In this file, we will write the actions that will call the respective mutations: +In this file, we will write the actions that will call mutations for handling a list of users: ```javascript import * as types from './mutation_types'; import axios from '~/lib/utils/axios_utils'; import createFlash from '~/flash'; - export const requestUsers = ({ commit }) => commit(types.REQUEST_USERS); - export const receiveUsersSuccess = ({ commit }, data) => commit(types.RECEIVE_USERS_SUCCESS, data); - export const receiveUsersError = ({ commit }, error) => commit(types.RECEIVE_USERS_ERROR, error); - export const fetchUsers = ({ state, dispatch }) => { - dispatch('requestUsers'); + commit(types.REQUEST_USERS); axios.get(state.endpoint) - .then(({ data }) => dispatch('receiveUsersSuccess', data)) + .then(({ data }) => commit(types.RECEIVE_USERS_SUCCESS, data)) .catch((error) => { - dispatch('receiveUsersError', error) + commit(types.RECEIVE_USERS_ERROR, error) createFlash('There was an error') }); } - export const requestAddUser = ({ commit }) => commit(types.REQUEST_ADD_USER); - export const receiveAddUserSuccess = ({ commit }, data) => commit(types.RECEIVE_ADD_USER_SUCCESS, data); - export const receiveAddUserError = ({ commit }, error) => commit(types.REQUEST_ADD_USER_ERROR, error); - export const addUser = ({ state, dispatch }, user) => { - dispatch('requestAddUser'); + commit(types.REQUEST_ADD_USER); axios.post(state.endpoint, user) - .then(({ data }) => dispatch('receiveAddUserSuccess', data)) - .catch((error) => dispatch('receiveAddUserError', error)); + .then(({ data }) => commit(types.RECEIVE_ADD_USER_SUCCESS, data)) + .catch((error) => commit(types.REQUEST_ADD_USER_ERROR, error)); } ``` -#### Actions Pattern: `request` and `receive` namespaces - -When a request is made we often want to show a loading state to the user. - -Instead of creating an action to toggle the loading state and dispatch it in the component, -create: - -1. An action `requestSomething`, to toggle the loading state -1. An action `receiveSomethingSuccess`, to handle the success callback -1. An action `receiveSomethingError`, to handle the error callback -1. An action `fetchSomething` to make the request. - 1. In case your application does more than a `GET` request you can use these as examples: - - `POST`: `createSomething` - - `PUT`: `updateSomething` - - `DELETE`: `deleteSomething` - -The component MUST only dispatch the `fetchNamespace` action. Actions namespaced with `request` or `receive` should not be called from the component -The `fetch` action will be responsible to dispatch `requestNamespace`, `receiveNamespaceSuccess` and `receiveNamespaceError` - -By following this pattern we guarantee: - -1. All applications follow the same pattern, making it easier for anyone to maintain the code -1. All data in the application follows the same lifecycle pattern -1. Actions are contained and human friendly -1. Unit tests are easier -1. Actions are simple and straightforward - #### Dispatching actions To dispatch an action from a component, use the `mapActions` helper: @@ -181,6 +145,8 @@ Remember that actions only describe that something happened, they don't describe **Never commit a mutation directly from a component** +Instead, you should create an action that will commit a mutation. + ```javascript import * as types from './mutation_types'; @@ -210,6 +176,31 @@ Remember that actions only describe that something happened, they don't describe }; ``` +#### Naming Pattern: `REQUEST` and `RECEIVE` namespaces + +When a request is made we often want to show a loading state to the user. + +Instead of creating an mutation to toggle the loading state, we should: + +1. A mutation with type `REQUEST_SOMETHING`, to toggle the loading state +1. A mutation with type `RECEIVE_SOMETHING_SUCCESS`, to handle the success callback +1. A mutation with type `RECEIVE_SOMETHING_ERROR`, to handle the error callback +1. An action `fetchSomething` to make the request and commit mutations on mentioned cases + 1. In case your application does more than a `GET` request you can use these as examples: + - `POST`: `createSomething` + - `PUT`: `updateSomething` + - `DELETE`: `deleteSomething` + +As a result, we can dispatch the `fetchNamespace` action from the component and it will be responsible to commit `REQUEST_NAMESPACE`, `RECEIVE_NAMESPACE_SUCCESS` and `RECEIVE_NAMESPACE_ERROR` mutations. + +> Previously, we were dispatching actions from the `fetchNamespace` action instead of committing mutation, so please don't be confused if you find a different pattern in the older parts of the codebase. However, we encourage leveraging a new pattern whenever you write new Vuex stores + +By following this pattern we guarantee: + +1. All applications follow the same pattern, making it easier for anyone to maintain the code +1. All data in the application follows the same lifecycle pattern +1. Unit tests are easier + ### `getters.js` Sometimes we may need to get derived state based on store state, like filtering for a specific prop. @@ -477,8 +468,6 @@ To prevent this error from happening, you need to export an empty function as `d export default () => {}; ``` -[vuex-docs]: https://vuex.vuejs.org - ### Two way data binding When storing form data in Vuex, it is sometimes necessary to update the value stored. The store should never be mutated directly, and an action should be used instead. diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index aa32285b27b..309aecd7978 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -16,32 +16,6 @@ run: /chatops run feature --help ``` -## Where to run commands - -To increase visibility, we recommend that GitLab team members run feature flag -related Chatops commands within certain Slack channels based on the environment -and related feature. For the [staging](https://staging.gitlab.com) -and [development](https://dev.gitlab.org) environments of GitLab.com, -the commands should run in a channel for the stage the feature is relevant too. - -For example, use the `#s_monitor` channel for features developed by the -Monitor stage, Health group. - -For all production environment Chatops commands, use the `#production` channel. - -As per the template, where a feature would have a (potentially) significant user -impact and the feature is being enabled instance wide prior to release, please copy -the Slack message and repost in the `#support_gitlab-com` channel for added visibility -and awareness, preferably with a link to the issue, MR, or docs. - -Regardless of the channel in which the Chatops command is ran, any feature flag change that affects GitLab.com will automatically be logged in an issue. - -The issue is created in the [gl-infra/feature-flag-log](https://gitlab.com/gitlab-com/gl-infra/feature-flag-log/issues?scope=all&utf8=%E2%9C%93&state=closed) project, and it will at minimum log the Slack handle of person enabling a feature flag, the time, and the name of the flag being changed. - -The issue is then also posted to GitLab Inc. internal [Grafana dashboard](https://dashboards.gitlab.net/) as an annotation marker to make the change even more visible. - -Changes to the issue format can be submitted in the [Chatops project](https://gitlab.com/gitlab-com/chatops). - ## Rolling out changes When the changes are deployed to the environments it is time to start @@ -67,19 +41,11 @@ If you get an error "Whoops! This action is not allowed. This incident will be reported." that means your Slack account is not allowed to change feature flags or you do not [have access](#access). -### Enabling feature for preproduction testing +### Enabling a feature for preproduction testing As a first step in a feature rollout, you should enable the feature on <https://staging.gitlab.com> and <https://dev.gitlab.org>. -For example, to enable a feature for 25% of all users, run the following in -Slack: - -```shell -/chatops run feature set new_navigation_bar 25 --dev -/chatops run feature set new_navigation_bar 25 --staging -``` - These two environments have different scopes. `dev.gitlab.org` is a production CE environment that has internal GitLab Inc. traffic and is used for some development and other related work. @@ -89,13 +55,65 @@ a (very) rough estimate of how your feature will look/behave on GitLab.com. Both of these instances are connected to Sentry so make sure you check the projects there for any exceptions while testing your feature after enabling the feature flag. -Once you are confident enough that these environments are in a good state with your -feature enabled, you can roll out the change to GitLab.com. +For these preproduction environments, the commands should be run in a +Slack channel for the stage the feature is relevant to. For example, use the +`#s_monitor` channel for features developed by the Monitor stage, Health +group. + +To enable a feature for 25% of all users, run the following in Slack: + +```shell +/chatops run feature set new_navigation_bar 25 --dev +/chatops run feature set new_navigation_bar 25 --staging +``` ### Enabling a feature for GitLab.com -Similar to above, to enable a feature for 25% of all users, run the following in -Slack: +When a feature has successfully been +[enabled on a preproduction](#enabling-a-feature-for-preproduction-testing) +environment and verified as safe and working, you can roll out the +change to GitLab.com (production). + +#### Communicate the change + +Some feature flag changes on GitLab.com should be communicated with +parts of the company. The developer responsible needs to determine +whether this is necessary and the appropriate level of communication. +This depends on the feature and what sort of impact it might have. + +As a guideline: + +- For simple features that are low-risk, and easily rolled back, then + just proceed to [enabling the feature in `#production`](#process). +- For features that will impact user experience consider notifying + `#support_gitlab-com` beforehand. +- For features with significant downstream effects (e.g.: turning on/off + Elasticsearch indexing) consider coordinating with `#production` + beforehand. + +#### Process + +Before toggling any feature flag, check that there are no ongoing +significant incidents on GitLab.com. You can do this by checking the +`#production` and `#incident-management` Slack channels, or looking for +[open incident issues](https://gitlab.com/gitlab-com/gl-infra/production/issues/?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=incident) +(although check the dates and times). + +We do not want to introduce changes during an incident, as it can make +diagnosis and resolution of the incident much harder to achieve, and +also will largely invalidate your rollout process as you will be unable +to assess whether the rollout was without problems or not. + +If there is any doubt, ask in `#production`. + +The following `/chatops` commands should be performed in the Slack +`#production` channel. + +When you begin to enable the feature, please link to the relevant +Feature Flag Rollout Issue within a Slack thread of the first `/chatops` +command you make so people can understand the change if they need to. + +To enable a feature for 25% of all users, run the following in Slack: ```shell /chatops run feature set new_navigation_bar 25 @@ -150,6 +168,23 @@ NOTE: **Note:** **Percentage of time** rollout is not a good idea if what you want is to make sure a feature is always on or off to the users. +### Feature flag change logging + +Any feature flag change that affects GitLab.com (production) will +automatically be logged in an issue. + +The issue is created in the +[gl-infra/feature-flag-log](https://gitlab.com/gitlab-com/gl-infra/feature-flag-log/issues?scope=all&utf8=%E2%9C%93&state=closed) +project, and it will at minimum log the Slack handle of person enabling +a feature flag, the time, and the name of the flag being changed. + +The issue is then also posted to GitLab's internal +[Grafana dashboard](https://dashboards.gitlab.net/) as an annotation +marker to make the change even more visible. + +Changes to the issue format can be submitted in the +[Chatops project](https://gitlab.com/gitlab-com/chatops). + ## Cleaning up Once the change is deemed stable, submit a new merge request to remove the @@ -157,7 +192,7 @@ feature flag. This ensures the change is available to all users and self-managed instances. Make sure to add the ~"feature flag" label to this merge request so release managers are aware the changes are hidden behind a feature flag. If the merge request has to be picked into a stable branch, make sure to also add the -appropriate "Pick into X" label (e.g. "Pick into XX.X"). +appropriate `~"Pick into X.Y"` label (e.g. `~"Pick into 13.0"`). See [the process document](process.md#including-a-feature-behind-feature-flag-in-the-final-release) for further details. When a feature gate has been removed from the code base, the feature diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index f5915f2c0a8..bd0bd8f2018 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -1,6 +1,7 @@ # Feature flags in development of GitLab -Feature flags can be used to gradually roll out changes, be +[Feature Flags](../../user/project/operations/feature_flags.md) +can be used to gradually roll out changes, be it a new feature, or a performance improvement. By using feature flags, we can comfortably measure the impact of our changes, while still being able to easily disable those changes, without having to revert an entire release. @@ -10,6 +11,5 @@ Before using feature flags for GitLab's development, read through the following: - [Process for using features flags](process.md). - [Developing with feature flags](development.md). - [Controlling feature flags](controls.md). - -When documenting feature flags, see [Feature flags](../documentation/styleguide.md#feature-flags) -in the Documentation Style Guide. +- [Documenting features deployed behind feature flags](../documentation/feature_flags.md). +- [How GitLab administrators can enable and disable features behind flags](../../administration/feature_flags.md). diff --git a/doc/development/feature_flags/process.md b/doc/development/feature_flags/process.md index 0cca4117f1f..57360f5b771 100644 --- a/doc/development/feature_flags/process.md +++ b/doc/development/feature_flags/process.md @@ -63,6 +63,9 @@ from when the merge request is first reviewed to when the change is deployed to GitLab.com. However, it is recommended to allow 10-14 days for this activity to account for unforeseen problems. +Feature flags must be [documented according to their state (enabled/disabled)](../documentation/feature_flags.md), +and when the state changes, docs **must** be updated accordingly. + NOTE: **Note:** Take into consideration that such action can make the feature available on GitLab.com shortly after the change to the feature flag is merged. diff --git a/doc/development/file_storage.md b/doc/development/file_storage.md index 230288844d7..e8ae5a11d48 100644 --- a/doc/development/file_storage.md +++ b/doc/development/file_storage.md @@ -21,7 +21,7 @@ There are many places where file uploading is used, according to contexts: - CI Artifacts (archive, metadata, trace) - LFS Objects - Merge request diffs - - Design Management design thumbnails (EE) + - Design Management design thumbnails ## Disk storage @@ -30,18 +30,18 @@ they are still not 100% standardized. You can see them below: | Description | In DB? | Relative path (from CarrierWave.root) | Uploader class | model_type | | ------------------------------------- | ------ | ----------------------------------------------------------- | ---------------------- | ---------- | -| Instance logo | yes | uploads/-/system/appearance/logo/:id/:filename | `AttachmentUploader` | Appearance | -| Header logo | yes | uploads/-/system/appearance/header_logo/:id/:filename | `AttachmentUploader` | Appearance | -| Group avatars | yes | uploads/-/system/group/avatar/:id/:filename | `AvatarUploader` | Group | -| User avatars | yes | uploads/-/system/user/avatar/:id/:filename | `AvatarUploader` | User | -| User snippet attachments | yes | uploads/-/system/personal_snippet/:id/:random_hex/:filename | `PersonalFileUploader` | Snippet | -| Project avatars | yes | uploads/-/system/project/avatar/:id/:filename | `AvatarUploader` | Project | -| Issues/MR/Notes Markdown attachments | yes | uploads/:project_path_with_namespace/:random_hex/:filename | `FileUploader` | Project | -| Issues/MR/Notes Legacy Markdown attachments | no | uploads/-/system/note/attachment/:id/:filename | `AttachmentUploader` | Note | -| Design Management design thumbnails (EE) | yes | uploads/-/system/design_management/action/image_v432x230/:id/:filename | `DesignManagement::DesignV432x230Uploader` | DesignManagement::Action | -| CI Artifacts (CE) | yes | `shared/artifacts/:disk_hash[0..1]/:disk_hash[2..3]/:disk_hash/:year_:month_:date/:job_id/:job_artifact_id` (:disk_hash is SHA256 digest of project_id) | `JobArtifactUploader` | Ci::JobArtifact | -| LFS Objects (CE) | yes | shared/lfs-objects/:hex/:hex/:object_hash | `LfsObjectUploader` | LfsObject | -| External merge request diffs | yes | shared/external-diffs/merge_request_diffs/mr-:parent_id/diff-:id | `ExternalDiffUploader` | MergeRequestDiff | +| Instance logo | yes | `uploads/-/system/appearance/logo/:id/:filename` | `AttachmentUploader` | Appearance | +| Header logo | yes | `uploads/-/system/appearance/header_logo/:id/:filename` | `AttachmentUploader` | Appearance | +| Group avatars | yes | `uploads/-/system/group/avatar/:id/:filename` | `AvatarUploader` | Group | +| User avatars | yes | `uploads/-/system/user/avatar/:id/:filename` | `AvatarUploader` | User | +| User snippet attachments | yes | `uploads/-/system/personal_snippet/:id/:random_hex/:filename` | `PersonalFileUploader` | Snippet | +| Project avatars | yes | `uploads/-/system/project/avatar/:id/:filename` | `AvatarUploader` | Project | +| Issues/MR/Notes Markdown attachments | yes | `uploads/:project_path_with_namespace/:random_hex/:filename` | `FileUploader` | Project | +| Issues/MR/Notes Legacy Markdown attachments | no | `uploads/-/system/note/attachment/:id/:filename` | `AttachmentUploader` | Note | +| Design Management design thumbnails | yes | `uploads/-/system/design_management/action/image_v432x230/:id/:filename` | `DesignManagement::DesignV432x230Uploader` | DesignManagement::Action | +| CI Artifacts (CE) | yes | `shared/artifacts/:disk_hash[0..1]/:disk_hash[2..3]/:disk_hash/:year_:month_:date/:job_id/:job_artifact_id` (`:disk_hash` is SHA256 digest of `project_id`) | `JobArtifactUploader` | Ci::JobArtifact | +| LFS Objects (CE) | yes | `shared/lfs-objects/:hex/:hex/:object_hash` | `LfsObjectUploader` | LfsObject | +| External merge request diffs | yes | `shared/external-diffs/merge_request_diffs/mr-:parent_id/diff-:id` | `ExternalDiffUploader` | MergeRequestDiff | CI Artifacts and LFS Objects behave differently in CE and EE. In CE they inherit the `GitlabUploader` while in EE they inherit the `ObjectStorage` and store files in and S3 API compatible object store. diff --git a/doc/development/filtering_by_label.md b/doc/development/filtering_by_label.md index 6f6d7afc040..19dece0d5c9 100644 --- a/doc/development/filtering_by_label.md +++ b/doc/development/filtering_by_label.md @@ -80,7 +80,7 @@ it did not improve query performance. ## Attempt B: Denormalize using an array column Having [removed MySQL support in GitLab 12.1](https://about.gitlab.com/blog/2019/06/27/removing-mysql-support/), -using [PostgreSQL's arrays](https://www.postgresql.org/docs/9.6/arrays.html) became more +using [PostgreSQL's arrays](https://www.postgresql.org/docs/11/arrays.html) became more tractable as we didn't have to support two databases. We discussed denormalizing the `label_links` table for querying in [issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/issues/49651), @@ -91,7 +91,7 @@ and `epics`: `issues.label_ids` would be an array column of label IDs, and `issues.label_titles` would be an array of label titles. These array columns can be complemented with [GIN -indexes](https://www.postgresql.org/docs/9.6/gin-intro.html) to improve +indexes](https://www.postgresql.org/docs/11/gin-intro.html) to improve matching. ### Attempt B1: store label IDs for each object diff --git a/doc/development/geo.md b/doc/development/geo.md index b922fdfa119..bf56340f8ec 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -216,15 +216,11 @@ bundle exec rake geo:db:migrate Foreign Data Wrapper ([FDW](#fdw)) is used by the [Geo Log Cursor](#geo-log-cursor) and improves the performance of many synchronization operations. -FDW is a PostgreSQL extension ([`postgres_fdw`](https://www.postgresql.org/docs/current/postgres-fdw.html)) that is enabled within +FDW is a PostgreSQL extension ([`postgres_fdw`](https://www.postgresql.org/docs/11/postgres-fdw.html)) that is enabled within the Geo Tracking Database (on a **secondary** node), which allows it to connect to the readonly database replica and perform queries and filter data from both instances. -While FDW is available in older versions of PostgreSQL, we needed to -raise the minimum required version to 9.6 as this includes many -performance improvements to the FDW implementation. - This persistent connection is configured as an FDW server named `gitlab_secondary`. This configuration exists within the database's user context only. To access the `gitlab_secondary`, GitLab needs to use the diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index 026d3543955..d72f7cc4cc1 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -84,10 +84,8 @@ module Geo model_record.file end - private - # Specify the model this replicator belongs to - def model + def self.model ::Packages::PackageFile end end @@ -163,47 +161,7 @@ state. For example, to add support for files referenced by a `Widget` model with a `widgets` table, you would perform the following steps: -1. Add verification state fields to the `widgets` table so the Geo primary can - track verification state: - - ```ruby - # frozen_string_literal: true - - class AddVerificationStateToWidgets < ActiveRecord::Migration[6.0] - DOWNTIME = false - - def change - add_column :widgets, :verification_retry_at, :datetime_with_timezone - add_column :widgets, :verified_at, :datetime_with_timezone - add_column :widgets, :verification_checksum, :string - add_column :widgets, :verification_failure, :string - add_column :widgets, :verification_retry_count, :integer - end - end - ``` - -1. Add a partial index on `verification_failure` to ensure re-verification can - be performed efficiently: - - ```ruby - # frozen_string_literal: true - - class AddVerificationFailureIndexToWidgets < ActiveRecord::Migration[6.0] - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - def up - add_concurrent_index :widgets, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial" - end - - def down - remove_concurrent_index :widgets, :verification_failure - end - end - ``` +#### Replication 1. Include `Gitlab::Geo::ReplicableModel` in the `Widget` class, and specify the Replicator class `with_replicator Geo::WidgetReplicator`. @@ -270,7 +228,7 @@ For example, to add support for files referenced by a `Widget` model with a ```ruby # frozen_string_literal: true - class CreateWidgetRegistry < ActiveRecord::Migration[5.2] + class CreateWidgetRegistry < ActiveRecord::Migration[6.0] DOWNTIME = false def change @@ -334,7 +292,7 @@ For example, to add support for files referenced by a `Widget` model with a end ``` -1. Create `ee/spec/models/geo/widget_registry.rb`: +1. Create `ee/spec/models/geo/widget_registry_spec.rb`: ```ruby # frozen_string_literal: true @@ -350,4 +308,206 @@ For example, to add support for files referenced by a `Widget` model with a end ``` -Widget files should now be replicated and verified by Geo! +Widgets should now be replicated by Geo! + +#### Verification + +1. Add verification state fields to the `widgets` table so the Geo primary can + track verification state: + + ```ruby + # frozen_string_literal: true + + class AddVerificationStateToWidgets < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + add_column :widgets, :verification_retry_at, :datetime_with_timezone + add_column :widgets, :verified_at, :datetime_with_timezone + add_column :widgets, :verification_checksum, :binary, using: 'verification_checksum::bytea' + add_column :widgets, :verification_failure, :string + add_column :widgets, :verification_retry_count, :integer + end + end + ``` + +1. Add a partial index on `verification_failure` and `verification_checksum` to ensure + re-verification can be performed efficiently: + + ```ruby + # frozen_string_literal: true + + class AddVerificationFailureIndexToWidgets < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :widgets, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial" + add_concurrent_index :widgets, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "widgets_verification_checksum_partial" + end + + def down + remove_concurrent_index :widgets, :verification_failure + remove_concurrent_index :widgets, :verification_checksum + end + end + ``` + +1. Add fields `widget_count`, `widget_checksummed_count`, and `widget_checksum_failed_count` + to `GeoNodeStatus#RESOURCE_STATUS_FIELDS` array in `ee/app/models/geo_node_status.rb`. +1. Add the same fields to `GeoNodeStatus#PROMETHEUS_METRICS` hash in + `ee/app/models/geo_node_status.rb`. +1. Add the same fields to `Sidekiq metrics` table in + `doc/administration/monitoring/prometheus/gitlab_metrics.md`. +1. Add the same fields to `GET /geo_nodes/status` example response in `doc/api/geo_nodes.md`. +1. Modify `GeoNodeStatus#load_verification_data` to make sure the fields mantioned above + are set: + + ```ruby + self.widget_count = Geo::WidgetReplicator.model.count + self.widget_checksummed_count = Geo::WidgetReplicator.checksummed.count + self.widget_checksum_failed_count = Geo::WidgetReplicator.checksum_failed.count + ``` + +1. Make sure `Widget` model has `checksummed` and `checksum_failed` scopes. +1. Update `ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json` with new fields. +1. Update `GeoNodeStatus#PROMETHEUS_METRICS` hash in `ee/app/models/geo_node_status.rb` with new fields. +1. Update `Sidekiq metrics` table in `doc/administration/monitoring/prometheus/gitlab_metrics.md` with new fields. +1. Update `GET /geo_nodes/status` example response in `doc/api/geo_nodes.md` with new fields. +1. Update `ee/spec/models/geo_node_status_spec.rb` and `ee/spec/factories/geo_node_statuses.rb` with new fields. + +To do: Add verification on secondaries. This should be done as part of +[Geo: Self Service Framework - First Implementation for Package File verification](https://gitlab.com/groups/gitlab-org/-/epics/1817) + +Widgets should now be verified by Geo! + +#### GraphQL API + +1. Add a new field to `GeoNodeType` in + `ee/app/graphql/types/geo/geo_node_type.rb`: + + ```ruby + field :widget_registries, ::Types::Geo::WidgetRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::WidgetRegistriesResolver, + description: 'Find widget registries on this Geo node', + feature_flag: :geo_self_service_framework + ``` + +1. Add the new `widget_registries` field name to the `expected_fields` array in + `ee/spec/graphql/types/geo/geo_node_type_spec.rb`. + +1. Create `ee/app/graphql/resolvers/geo/widget_registries_resolver.rb`: + + ```ruby + # frozen_string_literal: true + + module Resolvers + module Geo + class WidgetRegistriesResolver < BaseResolver + include RegistriesResolver + end + end + end + ``` + +1. Create `ee/spec/graphql/resolvers/geo/widget_registries_resolver_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + describe Resolvers::Geo::WidgetRegistriesResolver do + it_behaves_like 'a Geo registries resolver', :widget_registry + end + ``` + +1. Create `ee/app/finders/geo/widget_registry_finder.rb`: + + ```ruby + # frozen_string_literal: true + + module Geo + class WidgetRegistryFinder + include FrameworkRegistryFinder + end + end + ``` + +1. Create `ee/spec/finders/geo/widget_registry_finder_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + describe Geo::WidgetRegistryFinder do + it_behaves_like 'a framework registry finder', :widget_registry + end + ``` + +1. Create `ee/app/graphql/types/geo/package_file_registry_type.rb`: + + ```ruby + # frozen_string_literal: true + + module Types + module Geo + # rubocop:disable Graphql/AuthorizeTypes because it is included + class WidgetRegistryType < BaseObject + include ::Types::Geo::RegistryType + + graphql_name 'WidgetRegistry' + description 'Represents the sync and verification state of a widget' + + field :widget_id, GraphQL::ID_TYPE, null: false, description: 'ID of the Widget' + end + end + end + ``` + +1. Create `ee/spec/graphql/types/geo/widget_registry_type_spec.rb`: + + ```ruby + # frozen_string_literal: true + + require 'spec_helper' + + describe GitlabSchema.types['WidgetRegistry'] do + it_behaves_like 'a Geo registry type' + + it 'has the expected fields (other than those included in RegistryType)' do + expected_fields = %i[widget_id] + + expect(described_class).to have_graphql_fields(*expected_fields).at_least + end + end + ``` + +1. Add integration tests for providing Widget registry data to the frontend via + the GraphQL API, by duplicating and modifying the following shared examples + in `ee/spec/requests/api/graphql/geo/registries_spec.rb`: + + ```ruby + it_behaves_like 'gets registries for', { + field_name: 'widgetRegistries', + registry_class_name: 'WidgetRegistry', + registry_factory: :widget_registry, + registry_foreign_key_field_name: 'widgetId' + } + ``` + +Individual widget synchronization and verification data should now be available +via the GraphQL API! + +#### Admin UI + +To do: This should be done as part of +[Geo: Implement frontend for Self-Service Framework replicables](https://gitlab.com/groups/gitlab-org/-/epics/2525) + +Widget sync and verification data (aggregate and individual) should now be +available in the Admin UI! diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 43e4d4d0e72..5e5cae7228b 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -5,7 +5,7 @@ Workhorse and GitLab-Shell. ## Deep Dive -In May 2019, Bob Van Landuyt hosted a [Deep Dive](https://gitlab.com/gitlab-org/create-stage/issues/1) +In May 2019, Bob Van Landuyt hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on GitLab's [Gitaly project](https://gitlab.com/gitlab-org/gitaly) and how to contribute to it as a Ruby developer, to share his domain specific knowledge with anyone who may work in this part of the code base in the future. @@ -77,7 +77,7 @@ If your test-suite is failing with Gitaly issues, as a first step, try running: rm -rf tmp/tests/gitaly ``` -During rspec tests, the Gitaly instance will write logs to `gitlab/log/gitaly-test.log`. +During RSpec tests, the Gitaly instance will write logs to `gitlab/log/gitaly-test.log`. ## Legacy Rugged code diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index c1dfb220df8..fe69a4205f8 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -63,15 +63,20 @@ file and ask your manager to review and merge. ```yaml projects: gitlab: reviewer go - gitlab-foss: reviewer go ``` ## Code style and format - Avoid global variables, even in packages. By doing so you will introduce side effects if the package is included multiple times. -- Use `go fmt` before committing ([Gofmt](https://golang.org/cmd/gofmt/) is a - tool that automatically formats Go source code). +- Use `goimports` before committing. + [goimports](https://godoc.org/golang.org/x/tools/cmd/goimports) + is a tool that automatically formats Go source code using + [Gofmt](https://golang.org/cmd/gofmt/), in addition to formatting import lines, + adding missing ones and removing unreferenced ones. + + Most editors/IDEs will allow you to run commands before/after saving a file, you can set it + up to run `goimports` so that it's applied to every file when saving. - Place private methods below the first caller method in the source file. ### Automatic linting @@ -244,6 +249,59 @@ Programs handling a lot of IO or complex operations should always include [benchmarks](https://golang.org/pkg/testing/#hdr-Benchmarks), to ensure performance consistency over time. +## Error handling + +### Adding context + +Adding context before you return the error can be helpful, instead of +just returning the error. This allows developers to understand what the +program was trying to do when it entered the error state making it much +easier to debug. + +For example: + +```go +// Wrap the error +return nil, fmt.Errorf("get cache %s: %w", f.Name, err) + +// Just add context +return nil, fmt.Errorf("saving cache %s: %v", f.Name, err) +``` + +A few things to keep in mind when adding context: + +- Decide if you want to expose the underlying error + to the caller. If so, use `%w`, if not, you can use `%v`. +- Don't use words like `failed`, `error`, `didn't`. As it's an error, + the user already knows that something failed and this might lead to + having strings like `failed xx failed xx failed xx`. Explain _what_ + failed instead. +- Error strings should not be capitalized or end with punctuation or a + newline. You can use `golint` to check for this. + +### Naming + +- When using sentinel errors they should always be named like `ErrXxx`. +- When creating a new error type they should always be named like + `XxxError`. + +### Checking Error types + +- To check error equality don't use `==`. Use + [`errors.Is`](https://pkg.go.dev/errors?tab=doc#Is) instead (for Go + versions >= 1.13). +- To check if the error is of a certain type don't use type assertion, + use [`errors.As`](https://pkg.go.dev/errors?tab=doc#As) instead (for + Go versions >= 1.13). + +### References for working with errors + +- [Go 1.13 errors](https://blog.golang.org/go1.13-errors). +- [Programing with + errors](https://peter.bourgon.org/blog/2019/09/11/programming-with-errors.html). +- [Don’t just check errors, handle them + gracefully](https://dave.cheney.net/2016/04/27/dont-just-check-errors-handle-them-gracefully). + ## CLIs Every Go program is launched from the command line. @@ -368,13 +426,13 @@ Once you've picked a new Go version to use, the steps to update Omnibus and CNG are: - [Create a merge request in the CNG project](https://gitlab.com/gitlab-org/build/CNG/edit/master/ci_files/variables.yml?branch_name=update-go-version), - updating the `GO_VERSION` in `ci_files/variables.yml`. + updating the `GO_VERSION` in `ci_files/variables.yml`. - Create a merge request in the [`gitlab-omnibus-builder` project](https://gitlab.com/gitlab-org/gitlab-omnibus-builder), - updating every file in the `docker/` directory so the `GO_VERSION` is set - appropriately. [Here's an example](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/125/diffs). + updating every file in the `docker/` directory so the `GO_VERSION` is set + appropriately. [Here's an example](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/125/diffs). - Tag a new release of `gitlab-omnibus-builder` containing the change. -- [Create a merge request in the `gitlab-omnibus` project](https://gitlab.com/gitlab-org/omnibus-gitlab/edit/master/.gitlab-ci.yml?branch_name=update-gitlab-omnibus-builder-version), - updating the `BUILDER_IMAGE_REVISION` to match the newly-created tag. +- [Create a merge request in the `omnibus-gitlab` project](https://gitlab.com/gitlab-org/omnibus-gitlab/edit/master/.gitlab-ci.yml?branch_name=update-gitlab-omnibus-builder-version), + updating the `BUILDER_IMAGE_REVISION` to match the newly-created tag. To reduce unnecessary differences between two distribution methods, Omnibus and CNG **should always use the same Go version**. diff --git a/doc/development/hash_indexes.md b/doc/development/hash_indexes.md index 417ea18e22f..bc962ac0cd6 100644 --- a/doc/development/hash_indexes.md +++ b/doc/development/hash_indexes.md @@ -14,7 +14,7 @@ documentation: > answers to queries that subsequently use them. For these reasons, hash index > use is presently discouraged. -RuboCop is configured to register an offence when it detects the use of a hash +RuboCop is configured to register an offense when it detects the use of a hash index. Instead of using hash indexes you should use regular btree indexes. diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index 7ddcd426fd7..a81e656fc27 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -20,7 +20,7 @@ The following tools are used: 1. [`gettext_i18n_rails`](https://github.com/grosser/gettext_i18n_rails): this gem allow us to translate content from models, views and controllers. Also - it gives us access to the following raketasks: + it gives us access to the following Rake tasks: - `rake gettext:find`: Parses almost all the files from the Rails application looking for content that has been marked for translation. Finally, it updates the PO files with the new content that @@ -30,7 +30,7 @@ The following tools are used: 1. [`gettext_i18n_rails_js`](https://github.com/webhippie/gettext_i18n_rails_js): this gem is useful to make the translations available in JavaScript. It - provides the following raketask: + provides the following Rake task: - `rake gettext:po_to_json`: Reads the contents from the PO files and generates JSON files containing all the available translations. @@ -131,7 +131,7 @@ You can mark that content for translation with: In JavaScript we added the `__()` (double underscore parenthesis) function that you can import from the `~/locale` file. For instance: -```js +```javascript import { __ } from '~/locale'; const label = __('Subscribe'); ``` @@ -167,7 +167,7 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s - In JavaScript (when Vue cannot be used): - ```js + ```javascript import { __, sprintf } from '~/locale'; sprintf(__('Hello %{username}'), { username: 'Joe' }); // => 'Hello Joe' @@ -180,7 +180,7 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s escape any interpolated dynamic values yourself, for instance using `escape` from `lodash`. - ```js + ```javascript import { escape } from 'lodash'; import { __, sprintf } from '~/locale'; @@ -220,14 +220,14 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s - In JavaScript: - ```js + ```javascript n__('Apple', 'Apples', 3) // => 'Apples' ``` Using interpolation: - ```js + ```javascript n__('Last day', 'Last %d days', x) // => When x == 1: 'Last day' // => When x == 2: 'Last 2 days' @@ -274,7 +274,7 @@ Namespaces should be PascalCase. - In JavaScript: - ```js + ```javascript s__('OpenedNDaysAgo|Opened') ``` @@ -285,7 +285,7 @@ guidelines for more details](translation.md#namespaced-strings). - In JavaScript: -```js +```javascript import { createDateTimeFormat } from '~/locale'; const dateFormat = createDateTimeFormat({ year: 'numeric', month: 'long', day: 'numeric' }); @@ -372,7 +372,7 @@ structure is the same in all languages. For instance, the following: -```js +```javascript {{ s__("mrWidget|Set by") }} {{ author.name }} {{ s__("mrWidget|to be merged automatically when the pipeline succeeds") }} @@ -380,7 +380,7 @@ For instance, the following: should be externalized as follows: -```js +```javascript {{ sprintf(s__("mrWidget|Set by %{author} to be merged automatically when the pipeline succeeds"), { author: author.name }) }} ``` @@ -439,7 +439,7 @@ This also applies when using links in between translated sentences, otherwise th - In JavaScript (when Vue cannot be used), instead of: - ```js + ```javascript {{ sprintf(s__("ClusterIntegration|Learn more about %{link}"), { link: '<a href="https://cloud.google.com/compute/docs/regions-zones/regions-zones" target="_blank" rel="noopener noreferrer">zones</a>' @@ -449,7 +449,7 @@ This also applies when using links in between translated sentences, otherwise th Set the link starting and ending HTML fragments as placeholders like so: - ```js + ```javascript {{ sprintf(s__("ClusterIntegration|Learn more about %{linkStart}zones%{linkEnd}"), { linkStart: '<a href="https://cloud.google.com/compute/docs/regions-zones/regions-zones" target="_blank" rel="noopener noreferrer">', @@ -536,9 +536,9 @@ The linter will take the following into account: - Variable usage - Only one unnamed (`%d`) variable, since the order of variables might change in different languages - - All variables used in the message-id are used in the translation + - All variables used in the message ID are used in the translation - There should be no variables used in a translation that aren't in the - message-id + message ID - Errors during translation. The errors are grouped per file, and per message ID: @@ -562,7 +562,7 @@ Errors in `locale/zh_TW/gitlab.po`: In this output the `locale/zh_HK/gitlab.po` has syntax errors. The `locale/zh_TW/gitlab.po` has variables that are used in the translation that -aren't in the message with id `1 pipeline`. +aren't in the message with ID `1 pipeline`. ## Adding a new language diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index 1170103490b..837da349f7e 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -87,7 +87,7 @@ are very appreciative of the work done by translators and proofreaders! - Mark Minakou - [GitLab](https://gitlab.com/sandzhaj), [Crowdin](https://crowdin.com/profile/sandzhaj) - NickVolynkin - [Crowdin](https://crowdin.com/profile/NickVolynkin) - Andrey Komarov - [GitLab](https://gitlab.com/elkamarado), [Crowdin](https://crowdin.com/profile/kamarado) - - Iaroslav Postovalov - [GitLab](https://gitlab/CMDR_Tvis), [Crowdin](https://crowdin.com/profile/CMDR_Tvis) + - Iaroslav Postovalov - [GitLab](https://gitlab.com/CMDR_Tvis), [Crowdin](https://crowdin.com/profile/CMDR_Tvis) - Serbian (Cyrillic) - Proofreaders needed. - Serbian (Latin) diff --git a/doc/development/img/snowplow_flow.png b/doc/development/img/snowplow_flow.png Binary files differnew file mode 100644 index 00000000000..5996cf01537 --- /dev/null +++ b/doc/development/img/snowplow_flow.png diff --git a/doc/development/img/telemetry_system_overview.png b/doc/development/img/telemetry_system_overview.png Binary files differnew file mode 100644 index 00000000000..1667039e8cd --- /dev/null +++ b/doc/development/img/telemetry_system_overview.png diff --git a/doc/development/import_export.md b/doc/development/import_export.md index 5a7d176d1d3..6d1b6929667 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -331,3 +331,78 @@ module Projects wiki_repo_saver, lfs_saver].all?(&:save) end ``` + +## Test fixtures + +Fixtures used in Import/Export specs live in `spec/fixtures/lib/gitlab/import_export`. There are both Project and Group fixtures. + +There are two versions of each of these fixtures: + +- A human readable single JSON file with all objects, called either `project.json` or `group.json`. +- A folder named `tree`, containing a tree of files in `ndjson` format. **Please do not edit files under this folder manually unless strictly necessary.** + +The tools to generate the NDJSON tree from the human-readable JSON files live in the [`gitlab-org/memory-team/team-tools`](https://gitlab.com/gitlab-org/memory-team/team-tools/-/blob/master/import-export/) project. + +### Project + +**Please use `legacy-project-json-to-ndjson.sh` to generate the NDJSON tree.** + +The NDJSON tree will look like this: + +```shell +tree +├── project +│ ├── auto_devops.ndjson +│ ├── boards.ndjson +│ ├── ci_cd_settings.ndjson +│ ├── ci_pipelines.ndjson +│ ├── container_expiration_policy.ndjson +│ ├── custom_attributes.ndjson +│ ├── error_tracking_setting.ndjson +│ ├── external_pull_requests.ndjson +│ ├── issues.ndjson +│ ├── labels.ndjson +│ ├── merge_requests.ndjson +│ ├── milestones.ndjson +│ ├── pipeline_schedules.ndjson +│ ├── project_badges.ndjson +│ ├── project_feature.ndjson +│ ├── project_members.ndjson +│ ├── protected_branches.ndjson +│ ├── protected_tags.ndjson +│ ├── releases.ndjson +│ ├── services.ndjson +│ ├── snippets.ndjson +│ └── triggers.ndjson +└── project.json +``` + +### Group + +**Please use `legacy-group-json-to-ndjson.rb` to generate the NDJSON tree.** + +The NDJSON tree will look like this: + +```shell +tree +└── groups + ├── 4351 + │ ├── badges.ndjson + │ ├── boards.ndjson + │ ├── epics.ndjson + │ ├── labels.ndjson + │ ├── members.ndjson + │ └── milestones.ndjson + ├── 4352 + │ ├── badges.ndjson + │ ├── boards.ndjson + │ ├── epics.ndjson + │ ├── labels.ndjson + │ ├── members.ndjson + │ └── milestones.ndjson + ├── _all.ndjson + ├── 4351.json + └── 4352.json +``` + +CAUTION: **Caution:** When updating these fixtures, please ensure you update both `json` files and `tree` folder, as the tests apply to both. diff --git a/doc/development/import_project.md b/doc/development/import_project.md index 78efc6ce2ab..f222a6533e8 100644 --- a/doc/development/import_project.md +++ b/doc/development/import_project.md @@ -61,10 +61,9 @@ Parameters: | `namespace_path` | string | yes | Namespace path | | `project_path` | string | yes | Project name | | `archive_path` | string | yes | Path to the exported project tarball you want to import | -| `measurement_enabled` | boolean | no | Measure execution time, number of SQL calls and GC count | ```shell -bundle exec rake "gitlab:import_export:import[root, root, testingprojectimport, /path/to/file.tar.gz, true]" +bundle exec rake "gitlab:import_export:import[root, root, testingprojectimport, /path/to/file.tar.gz]" ``` ### Importing via the Rails console diff --git a/doc/development/instrumentation.md b/doc/development/instrumentation.md index fd01d0ea405..d72e1c6635e 100644 --- a/doc/development/instrumentation.md +++ b/doc/development/instrumentation.md @@ -5,7 +5,7 @@ blocks of Ruby code. Method instrumentation is the primary form of instrumentation with block-based instrumentation only being used when we want to drill down to specific regions of code within a method. -Please refer to [Telemetry](../telemetry/index.md) if you are tracking product usage patterns. +Please refer to [Telemetry](telemetry/index.md) if you are tracking product usage patterns. ## Instrumenting Methods diff --git a/doc/development/integrations/example_vuln.png b/doc/development/integrations/example_vuln.png Binary files differnew file mode 100644 index 00000000000..f7a3c8b38f2 --- /dev/null +++ b/doc/development/integrations/example_vuln.png diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index b38e45778fb..b0e1e28ba8b 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -2,14 +2,23 @@ Integrating a security scanner into GitLab consists of providing end users with a [CI job definition](../../ci/yaml/README.md#introduction) -they can add to their CI configuration files, to scan their GitLab projects. +they can add to their CI configuration files to scan their GitLab projects. +This CI job should then output its results in a GitLab-specified format. These results are then +automatically presented in various places in GitLab, such as the Pipeline view, Merge Request +widget, and Security Dashboard. + The scanning job is usually based on a [Docker image](https://docs.docker.com/) that contains the scanner and all its dependencies in a self-contained environment. -This page documents requirements and guidelines for writing CI jobs implementing a security scanner, -as well as requirements and guidelines for the Docker image itself. + +This page documents requirements and guidelines for writing CI jobs that implement a security +scanner, as well as requirements and guidelines for the Docker image. ## Job definition +This section desribes several important fields to add to the security scanner's job +definition file. Full documentation on these and other available fields can be viewed +in the [CI documentation](../../ci/yaml/README.md#image). + ### Name For consistency, scanning jobs should be named after the scanner, in lower case. @@ -26,8 +35,8 @@ containing the security scanner. ### Script The [`script`](../../ci/yaml/README.md#script) keyword -is used to specify the command that the job runs. -Because the `script` cannot be left empty, it must be set to the command that performs the scan. +is used to specify the commands to run the scanner. +Because the `script` entry can't be left empty, it must be set to the command that performs the scan. It is not possible to rely on the predefined `ENTRYPOINT` and `CMD` of the Docker image to perform the scan automatically, without passing any command. @@ -53,44 +62,41 @@ so the [`allow_failure`](../../ci/yaml/README.md#allow_failure) parameter should ### Artifacts Scanning jobs must declare a report that corresponds to the type of scanning they perform, -using the [`artifacts:reports`](../../ci/yaml/README.md#artifactsreports) keyword. +using the [`artifacts:reports`](../../ci/pipelines/job_artifacts.md#artifactsreports) keyword. Valid reports are: `dependency_scanning`, `container_scanning`, `dast`, and `sast`. For example, here is the definition of a SAST job that generates a file named `gl-sast-report.json`, and uploads it as a SAST report: ```yaml -mysec_dependency_scanning: +mysec_sast_scanning: image: registry.gitlab.com/secure/mysec artifacts: reports: sast: gl-sast-report.json ``` -`gl-sast-report.json` is an example file path. See [the Output file section](#output-file) for more details. -It is processed as a SAST report because it is declared as such in the job definition. +Note that `gl-sast-report.json` is an example file path but any other file name can be used. See +[the Output file section](#output-file) for more details. It's processed as a SAST report because +it's declared under the `reports:sast` key in the job definition, not because of the file name. ### Policies -Scanning jobs should be skipped unless the corresponding feature is listed -in the `GITLAB_FEATURES` variable (comma-separated list of values). -So Dependency Scanning, Container Scanning, SAST, and DAST should be skipped -unless `GITLAB_FEATURES` contains `dependency_scanning`, `container_scanning`, `sast`, and `dast`, respectively. -See [GitLab CI/CD predefined variables](../../ci/variables/predefined_variables.md). - -Also, scanning jobs should be skipped when the corresponding variable prefixed with `_DISABLED` is present. -See `DEPENDENCY_SCANNING_DISABLED`, `CONTAINER_SCANNING_DISABLED`, `SAST_DISABLED`, and `DAST_DISABLED` -in [Auto DevOps documentation](../../topics/autodevops/customize.md#disable-jobs). +Certain GitLab workflows, such as [AutoDevOps](../../topics/autodevops/customize.md#disable-jobs), +define variables to indicate that given scans should be disabled. You can check for this by looking +for variables such as `DEPENDENCY_SCANNING_DISABLED`, `CONTAINER_SCANNING_DISABLED`, +`SAST_DISABLED`, and `DAST_DISABLED`. If appropriate based on the scanner type, you should then +disable running the custom scanner. -Finally, SAST and Dependency Scanning job definitions should use -`CI_PROJECT_REPOSITORY_LANGUAGES` (comma-separated list of values) -in order to skip the job when the language or technology is not supported. +GitLab also defines a `CI_PROJECT_REPOSITORY_LANGUAGES` variable, which provides the list of +languages in the repo. Depending on this value, your scanner may or may not do something different. Language detection currently relies on the [`linguist`](https://github.com/github/linguist) Ruby gem. See [GitLab CI/CD prefined variables](../../ci/variables/predefined_variables.md#variables-reference). -For instance, here is how to skip the Dependency Scanning job `mysec_dependency_scanning` -unless the project repository contains Java source code, -and the `dependency_scanning` feature is enabled: +#### Policy checking example + +This example shows how to skip a custom Dependency Scanning job, `mysec_dependency_scanning`, unless +the project repository contains Java source code and the `dependency_scanning` feature is enabled: ```yaml mysec_dependency_scanning: @@ -111,6 +117,8 @@ for a particular branch or when a particular set of files changes. The Docker image is a self-contained environment that combines the scanner with all the libraries and tools it depends on. +Packaging your scanner into a Docker image makes its dependencies and configuration always present, +regardless of the individual machine the scanner runs on. ### Image size @@ -144,7 +152,7 @@ It also generates text output on the standard output and standard error streams, All CI variables are passed to the scanner as environment variables. The scanned project is described by the [predefined CI variables](../../ci/variables/README.md). -#### SAST, Dependency Scanning +#### SAST and Dependency Scanning SAST and Dependency Scanning scanners must scan the files in the project directory, given by the `CI_PROJECT_DIR` variable. @@ -178,7 +186,7 @@ It is recommended to name the output file after the type of scanning, and to use Since all Secure reports are JSON files, it is recommended to use `.json` as a file extension. For instance, a suggested file name for a Dependency Scanning report is `gl-dependency-scanning.json`. -The [`artifacts:reports`](../../ci/yaml/README.md#artifactsreports) keyword +The [`artifacts:reports`](../../ci/pipelines/job_artifacts.md#artifactsreports) keyword of the job definition must be consistent with the file path where the Security report is written. For instance, if a Dependency Scanning analyzer writes its report to the CI project directory, and if this report file name is `depscan.json`, @@ -223,11 +231,8 @@ The DAST variant of the report JSON format is not documented at the moment. ### Version -The documentation of -[SAST](../../user/application_security/sast/index.md#reports-json-format), -[Dependency Scanning](../../user/application_security/dependency_scanning/index.md#reports-json-format), -and [Container Scanning](../../user/application_security/container_scanning/index.md#reports-json-format) -describes the Secure report format version. +This field specifies the version of the report schema you are using. Please reference individual scanner +pages for the specific versions to use. ### Vulnerabilities @@ -251,12 +256,17 @@ The `id` should not collide with any other scanner another integrator would prov #### Name, message, and description -The `name` and `message` fields contain a short description of the vulnerability, -whereas the `description` field provides more details. +The `name` and `message` fields contain a short description of the vulnerability. +The `description` field provides more details. -The `name` is context-free and contains no information on where the vulnerability has been found, +The `name` field is context-free and contains no information on where the vulnerability has been found, whereas the `message` may repeat the location. +As a visual example, this screenshot highlights where these fields are used when viewing a +vulnerability as part of a pipeline view. + + + For instance, a `message` for a vulnerability reported by Dependency Scanning gives information on the vulnerable dependency, which is redundant with the `location` field of the vulnerability. @@ -288,21 +298,17 @@ It should not repeat the other fields of the vulnerability object. In particular, the `description` should not repeat the `location` (what is affected) or the `solution` (how to mitigate the risk). -There is a proposal to remove either the `name` or the `message`, to remove ambiguities. -See [issue #36779](https://gitlab.com/gitlab-org/gitlab/issues/36779). - #### Solution -The `solution` field may contain instructions users should follow to fix the vulnerability or to mitigate the risk. -It is intended for users whereas the `remediations` objects are processed automatically by GitLab. +You can use the `solution` field to instruct users how to fix the identified vulnerability or to mitigate +the risk. End-users interact with this field, whereas GitLab automatically processes the +`remediations` objects. #### Identifiers -The `identifiers` array describes the vulnerability flaw that has been detected. -An identifier object has a `type` and a `value`; -these technical fields are used to tell if two identifiers are the same. -It also has a `name` and a `url`; -these fields are used to display the identifier in the user interface. +The `identifiers` array describes the detected vulnerability. An identifier object's `type` and +`value` fields are used to tell if two identifiers are the same. The user interface uses the +object's `name` and `url` fields to display the identifier. It is recommended to reuse the identifiers the GitLab scanners already define: @@ -316,18 +322,15 @@ It is recommended to reuse the identifiers the GitLab scanners already define: | [RHSA](https://access.redhat.com/errata/#/) | `rhsa` | RHSA-2020:0111 | | [ELSA](https://linux.oracle.com/security/) | `elsa` | ELSA-2020-0085 | -The generic identifiers listed above are defined in the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common); -this library is shared by the analyzers maintained by GitLab, -and this is where you can [contribute](https://gitlab.com/gitlab-org/security-products/analyzers/common/blob/master/issue/identifier.go) new generic identifiers. -Analyzers may also produce vendor-specific or product-specific identifiers; -these do not belong to the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common). +The generic identifiers listed above are defined in the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common), +which is shared by the analyzers that GitLab maintains. You can [contribute](https://gitlab.com/gitlab-org/security-products/analyzers/common/blob/master/issue/identifier.go) +new generic identifiers to if needed. Analyzers may also produce vendor-specific or product-specific +identifiers, which don't belong in the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common). The first item of the `identifiers` array is called the primary identifier. The primary identifier is particularly important, because it is used to -[track vulnerabilities](#tracking-merging-vulnerabilities) -as new commits are pushed to the repository. - -Identifiers are used to [merge duplicate vulnerabilities](#tracking-merging-vulnerabilities) +[track vulnerabilities](#tracking-and-merging-vulnerabilities) as new commits are pushed to the repository. +Identifiers are also used to [merge duplicate vulnerabilities](#tracking-and-merging-vulnerabilities) reported for the same commit, except for `CWE` and `WASC`. ### Location @@ -336,7 +339,7 @@ The `location` indicates where the vulnerability has been detected. The format of the location depends on the type of scanning. Internally GitLab extracts some attributes of the `location` to generate the **location fingerprint**, -which is used to [track vulnerabilities](#tracking-merging-vulnerabilities) +which is used to track vulnerabilities as new commits are pushed to the repository. The attributes used to generate the location fingerprint also depend on the type of scanning. @@ -426,12 +429,12 @@ combines `file`, `start_line`, and `end_line`, so these attributes are mandatory. All other attributes are optional. -### Tracking, merging vulnerabilities +### Tracking and merging vulnerabilities Users may give feedback on a vulnerability: -- they may dismiss a vulnerability if it does not apply to their projects -- or they may create an issue for a vulnerability, if there is a possible threat +- They may dismiss a vulnerability if it doesn't apply to their projects +- They may create an issue for a vulnerability if there's a possible threat GitLab tracks vulnerabilities so that user feedback is not lost when new Git commits are pushed to the repository. @@ -470,18 +473,57 @@ Valid values are: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, ### Remediations The `remediations` field of the report is an array of remediation objects. -Each remediation describes a patch that can be applied to automatically fix +Each remediation describes a patch that can be applied to +[automatically fix](../../user/application_security/#solutions-for-vulnerabilities-auto-remediation) a set of vulnerabilities. +Here is an example of a report that contains remediations. + +```json +{ + "vulnerabilities": [ + { + "category": "dependency_scanning", + "name": "Regular Expression Denial of Service", + "id": "123e4567-e89b-12d3-a456-426655440000", + "solution": "Upgrade to new versions.", + "scanner": { + "id": "gemnasium", + "name": "Gemnasium" + }, + "identifiers": [ + { + "type": "gemnasium", + "name": "Gemnasium-642735a5-1425-428d-8d4e-3c854885a3c9", + "value": "642735a5-1425-428d-8d4e-3c854885a3c9" + } + ] + } + ], + "remediations": [ + { + "fixes": [ + { + "id": "123e4567-e89b-12d3-a456-426655440000" + } + ], + "summary": "Upgrade to new version", + "diff": "ZGlmZiAtLWdpdCBhL3lhcm4ubG9jayBiL3lhcm4ubG9jawppbmRleCAwZWNjOTJmLi43ZmE0NTU0IDEwMDY0NAotLS0gYS95Y==" + } + ] +} +``` + #### Summary -The `summary` field is an overview of how the vulnerabilities can be fixed. +The `summary` field is an overview of how the vulnerabilities can be fixed. This field is required. #### Fixed vulnerabilities The `fixes` field is an array of objects that reference the vulnerabilities fixed by the -remediation. `fixes[].id` contains a fixed vulnerability's unique identifier. +remediation. `fixes[].id` contains a fixed vulnerability's [unique identifier](#id). This field is required. #### Diff -The `diff` field is a base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). +The `diff` field is a base64-encoded remediation code diff, compatible with +[`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). This field is required. diff --git a/doc/development/integrations/secure_partner_integration.md b/doc/development/integrations/secure_partner_integration.md index d8badda4125..59336b0e6a1 100644 --- a/doc/development/integrations/secure_partner_integration.md +++ b/doc/development/integrations/secure_partner_integration.md @@ -68,7 +68,7 @@ and complete an intgration with the Secure stage. 1. Ensure your pipeline jobs create a report artifact that GitLab can process to successfully display your own product's results with the rest of GitLab. - See detailed [technical directions](secure.md) for this step. - - Read more about [job report artifacts](../../ci/yaml/README.md#artifactsreports). + - Read more about [job report artifacts](../../ci/pipelines/job_artifacts.md#artifactsreports). - Read about [job artifacts](../../user/project/pipelines/job_artifacts.md). - Your report artifact must be in one of our currently supported formats. For more information, see the [documentation on reports](secure.md#report). diff --git a/doc/development/interacting_components.md b/doc/development/interacting_components.md index 4639bd7cc20..697c64986b1 100644 --- a/doc/development/interacting_components.md +++ b/doc/development/interacting_components.md @@ -10,7 +10,7 @@ when making _backend_ changes that might involve multiple features or [component ## Uploads GitLab supports uploads to [object storage](https://docs.gitlab.com/charts/advanced/external-object-storage/). That means every feature and -change that affects uploads should also be tested against [object storage], +change that affects uploads should also be tested against [object storage](https://docs.gitlab.com/charts/advanced/external-object-storage/), which is _not_ enabled by default in [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit). When working on a related feature, make sure to enable and test it diff --git a/doc/development/internal_api.md b/doc/development/internal_api.md index e8fca7535b0..5b53f223eb0 100644 --- a/doc/development/internal_api.md +++ b/doc/development/internal_api.md @@ -40,7 +40,7 @@ POST /internal/allowed | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key_id` | string | no | Id of the SSH-key used to connect to GitLab-shell | +| `key_id` | string | no | ID of the SSH-key used to connect to GitLab-shell | | `username` | string | no | Username from the certificate used to connect to GitLab-Shell | | `project` | string | no (if `gl_repository` is passed) | Path to the project | | `gl_repository` | string | no (if `project` is passed) | Path to the project | @@ -93,7 +93,7 @@ information for LFS clients when the repository is accessed over SSH. | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key_id` | string | no | Id of the SSH-key used to connect to GitLab-shell | +| `key_id` | string | no | ID of the SSH-key used to connect to GitLab-shell | | `username`| string | no | Username from the certificate used to connect to GitLab-Shell | | `project` | string | no | Path to the project | @@ -151,14 +151,14 @@ Example response: - GitLab-shell -## Get user for user id or key +## Get user for user ID or key This endpoint is used when a user performs `ssh git@gitlab.com`. It discovers the user associated with an SSH key. | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key_id` | integer | no | The id of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | +| `key_id` | integer | no | The ID of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | | `username` | string | no | Username of the user being looked up, used by GitLab-shell when authenticating using a certificate | ```plaintext @@ -223,7 +223,7 @@ recovery codes based on their SSH key | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key_id` | integer | no | The id of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | +| `key_id` | integer | no | The ID of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | | `user_id` | integer | no | **Deprecated** User_id for which to generate new recovery codes | ```plaintext diff --git a/doc/development/lfs.md b/doc/development/lfs.md index e64bc0f7d3a..32e2e3d1bde 100644 --- a/doc/development/lfs.md +++ b/doc/development/lfs.md @@ -2,7 +2,7 @@ ## Deep Dive -In April 2019, Francisco Javier López hosted a [Deep Dive](https://gitlab.com/gitlab-org/create-stage/issues/1) +In April 2019, Francisco Javier López hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on GitLab's [Git LFS](../topics/git/lfs/index.md) implementation to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=Yyxwcksr0Qc), diff --git a/doc/development/licensed_feature_availability.md b/doc/development/licensed_feature_availability.md index 29e4ace157b..4f962b6f5e2 100644 --- a/doc/development/licensed_feature_availability.md +++ b/doc/development/licensed_feature_availability.md @@ -35,3 +35,7 @@ the instance license. ```ruby License.feature_available?(:feature_symbol) ``` + +## Enabling promo features on GitLab.com + +A paid feature can be made available to everyone on GitLab.com by enabling the feature flag `"promo_#{feature}"`. diff --git a/doc/development/licensing.md b/doc/development/licensing.md index 06a71967d06..e1be1faa61b 100644 --- a/doc/development/licensing.md +++ b/doc/development/licensing.md @@ -42,66 +42,6 @@ For all of the above, please include `--why "Reason"` and `--who "My Name"` so t More detailed information on how the gem and its commands work is available in the [License Finder README](https://github.com/pivotal/LicenseFinder). -## Acceptable Licenses +## Additional information -Libraries with the following licenses are acceptable for use: - -- [MIT License](https://choosealicense.com/licenses/mit/) (the MIT Expat License specifically): The MIT License requires that the license itself is included with all copies of the source. It is a permissive (non-copyleft) license as defined by the Open Source Initiative. -- [GNU Lesser General Public License (GNU LGPL)](https://choosealicense.com/licenses/lgpl-3.0/) (version 2, version 3): GPL constraints regarding modification and redistribution under the same license are not required of projects using an LGPL library, only upon modification of the LGPL-licensed library itself. -- [Apache 2.0 License](https://choosealicense.com/licenses/apache-2.0/): A permissive license that also provides an express grant of patent rights from contributors to users. -- [Ruby 1.8 License](https://github.com/ruby/ruby/blob/ruby_1_8_6/COPYING): Dual-licensed under either itself or the GPLv2, defer to the Ruby License itself. Acceptable because of point 3b: "You may distribute the software in object code or binary form, provided that you do at least ONE of the following: b) accompany the distribution with the machine-readable source of the software." -- [Ruby 1.9 License](https://www.ruby-lang.org/en/about/license.txt): Dual-licensed under either itself or the BSD 2-Clause License, defer to BSD 2-Clause. -- [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause): A permissive (non-copyleft) license as defined by the Open Source Initiative. -- [BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause) (also known as New BSD or Modified BSD): A permissive (non-copyleft) license as defined by the Open Source Initiative -- [ISC License](https://opensource.org/licenses/ISC) (also known as the OpenBSD License): A permissive (non-copyleft) license as defined by the Open Source Initiative. -- [Creative Commons Zero (CC0)](https://creativecommons.org/publicdomain/zero/1.0/): A public domain dedication, recommended as a way to disclaim copyright on your work to the maximum extent possible. -- [Unlicense](https://unlicense.org): Another public domain dedication. -- [OWFa 1.0](http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0): An open-source license and patent grant designed for specifications. -- [JSON License](https://www.json.org/license.html): Equivalent to the MIT license plus the statement, "The Software shall be used for Good, not Evil." - -## Unacceptable Licenses - -Libraries with the following licenses require legal approval for use: - -- [GNU GPL](https://choosealicense.com/licenses/gpl-3.0/) (version 1, [version 2](http://www.gnu.org/licenses/gpl-2.0.txt), [version 3](http://www.gnu.org/licenses/gpl-3.0.txt), or any future versions): GPL-licensed libraries cannot be linked to from non-GPL projects. -- [GNU AGPLv3](https://choosealicense.com/licenses/agpl-3.0/): AGPL-licensed libraries cannot be linked to from non-GPL projects. -- [Open Software License (OSL)](https://opensource.org/licenses/OSL-3.0): is a copyleft license. In addition, the FSF [recommend against its use](https://www.gnu.org/licenses/license-list.en.html#OSL). -- [WTFPL](https://wtfpl.net): is a public domain dedication [rejected by the OSI (3.2)](https://opensource.org/minutes20090304). Also has a strong language which is not in accordance with our diversity policy. - -## GPL Cooperation Commitment - -Before filing or continuing to prosecute any legal proceeding or claim (other than a Defensive Action) arising from termination of a Covered License, GitLab commits to extend to the person or entity (“you”) accused of violating the Covered License the following provisions regarding cure and reinstatement, taken from GPL version 3. As used here, the term ‘this License’ refers to the specific Covered License being enforced. - -However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. - -Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. - -GitLab intends this Commitment to be irrevocable, and binding and enforceable against GitLab and assignees of or successors to GitLab’s copyrights. - -GitLab may modify this Commitment by publishing a new edition on this page or a successor location. - -Definitions - -‘Covered License’ means the GNU General Public License, version 2 (GPLv2), the GNU Lesser General Public License, version 2.1 (LGPLv2.1), or the GNU Library General Public License, version 2 (LGPLv2), all as published by the Free Software Foundation. - -‘Defensive Action’ means a legal proceeding or claim that GitLab brings against you in response to a prior proceeding or claim initiated by you or your affiliate. - -GitLab means GitLab Inc. and its affiliates and subsidiaries. - -## Requesting Approval for Licenses or any other Intellectual Property - -Libraries that are not already approved and listed on the [Acceptable Licenses](#acceptable-licenses) list or that may be listed on the [Unacceptable Licenses](#unacceptable-licenses) list may be submitted to the legal team for review and use on a case-by-case basis. Please email `legal@gitlab.com` with the details of how the software will be used, whether or not it will be modified, and how it will be distributed (if at all). After a decision has been made, the original requestor is responsible for updating this document, if applicable. Not all approvals will be approved for universal use and may continue to remain on the Unacceptable License list. - -All inquiries relating to patents should be directed to the Legal team. - -## Notes - -Decisions regarding the GNU GPL licenses are based on information provided by [The GNU Project](http://www.gnu.org/licenses/gpl-faq.html#IfLibraryIsGPL), as well as [the Open Source Initiative](https://opensource.org/faq#linking-proprietary-code), which both state that linking GPL libraries makes the program itself GPL. - -If a gem uses a license which is not listed above, open an issue and ask. If a license is not included in the "acceptable" list, operate under the assumption that it is not acceptable. - -Keep in mind that each license has its own restrictions (typically defined in their body text). Please make sure to comply with those restrictions at all times whenever an external library is used. - -Dependencies which are only used in development or test environment are exempt from license requirements, as they're not distributed for use in production. - -**NOTE:** This document is **not** legal advice, nor is it comprehensive. It should not be taken as such. +Please see the [Open Source](https://about.gitlab.com/handbook/engineering/open-source/#using-open-source-libraries) page for more information on licensing. diff --git a/doc/development/logging.md b/doc/development/logging.md index ef2d2d7022d..e7d48e4d278 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -167,7 +167,8 @@ Resources: #### Logging durations Similar to timezones, choosing the right time unit to log can impose avoidable overhead. So, whenever -challenged to choose between seconds, milliseconds or any other unit, lean towards _seconds_ as float. +challenged to choose between seconds, milliseconds or any other unit, lean towards _seconds_ as float +(with microseconds precision, i.e. `Gitlab::InstrumentationHelper::DURATION_PRECISION`). In order to make it easier to track timings in the logs, make sure the log key has `_s` as suffix and `duration` within its name (e.g., `view_duration_s`). @@ -260,6 +261,8 @@ I, [2020-01-13T19:01:17.091Z #11056] INFO -- : {"message"=>"Message", "project_ lifecycle, which can then be added to the web request or Sidekiq logs. +The API, Rails and Sidekiq logs contain fields starting with `meta.` with this context information. + Entry points can be seen at: - [`ApplicationController`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/controllers/application_controller.rb) @@ -360,4 +363,4 @@ end project. See [this example](https://gitlab.com/gitlab-cookbooks/gitlab_fluentd/-/merge_requests/51/diffs). 1. Be sure to update the [GitLab CE/EE documentation](../administration/logs.md) and the [GitLab.com - runbooks](https://gitlab.com/gitlab-com/runbooks/blob/master/howto/logging.md). + runbooks](https://gitlab.com/gitlab-com/runbooks/blob/master/docs/logging/README.md). diff --git a/doc/development/mass_insert.md b/doc/development/mass_insert.md index 47f993a921e..c19850ca67e 100644 --- a/doc/development/mass_insert.md +++ b/doc/development/mass_insert.md @@ -1,7 +1,7 @@ -# Mass Inserting Rails Models +# Mass inserting Rails models -Setting the environment variable [`MASS_INSERT=1`](rake_tasks.md#env-variables) -when running `rake setup` will create millions of records, but these records +Setting the environment variable [`MASS_INSERT=1`](rake_tasks.md#environment-variables) +when running [`rake setup`](rake_tasks.md) will create millions of records, but these records aren't visible to the `root` user by default. To make any number of the mass-inserted projects visible to the `root` user, run diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index 4a9b5d26aef..b51fc681e27 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -24,7 +24,7 @@ The term `SHOULD` per the [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt) means > carefully weighed before choosing a different course. Ideally, each of these tradeoffs should be documented -in the separate issues, labelled accordingly and linked +in the separate issues, labeled accordingly and linked to original issue and epic. ## Impact Analysis diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 3e993243855..4cf546173de 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -35,16 +35,36 @@ and post-deployment migrations (`db/post_migrate`) are run after the deployment ## Schema Changes -Migrations that make changes to the database schema (e.g. adding a column) can -only be added in the monthly release, patch releases may only contain data -migrations _unless_ schema changes are absolutely required to solve a problem. +Changes to the schema should be commited to `db/structure.sql`. This +file is automatically generated by Rails, so you normally should not +edit this file by hand. If your migration is adding a column to a +table, that column will be added at the bottom. Please do not reorder +columns manually for existing tables as this will cause confusing to +other people using `db/structure.sql` generated by Rails. + +When your local database in your GDK is diverging from the schema from +`master` it might be hard to cleanly commit the schema changes to +Git. In that case you can use the `scripts/regenerate-schema` script to +regenerate a clean `db/structure.sql` for the migrations you're +adding. This script will apply all migrations found in `db/migrate` +or `db/post_migrate`, so if there are any migrations you don't want to +commit to the schema, rename or remove them. If your branch is not +targetting `master` you can set the `TARGET` environment variable. + +```shell +# Regenerate schema against `master` +scripts/regenerate-schema + +# Regenerate schema against `12-9-stable-ee` +TARGET=12-9-stable-ee scripts/regenerate-schema +``` ## What Requires Downtime? The document ["What Requires Downtime?"](what_requires_downtime.md) specifies various database operations, such as -- [adding, dropping, and renaming columns](what_requires_downtime.md#adding-columns) +- [dropping and renaming columns](what_requires_downtime.md#dropping-columns) - [changing column constraints and types](what_requires_downtime.md#changing-column-constraints) - [adding and dropping indexes, tables, and foreign keys](what_requires_downtime.md#adding-indexes) @@ -307,6 +327,34 @@ def down end ``` +**Usage with `disable_ddl_transaction!`** + +Generally the `with_lock_retries` helper should work with `disabled_ddl_transaction!`. A custom RuboCop rule ensures that only allowed methods can be placed within the lock retries block. + +```ruby +disable_ddl_transaction! + +def up + with_lock_retries do + add_column :users, :name, :text + end + + add_text_limit :users, :name, 255 # Includes constraint validation (full table scan) +end +``` + +The RuboCop rule generally allows standard Rails migration methods, listed below. This example will cause a rubocop offense: + +```ruby +disabled_ddl_transaction! + +def up + with_lock_retries do + add_concurrent_index :users, :name + end +end +``` + ### When to use the helper method The `with_lock_retries` helper method can be used when you normally use @@ -330,8 +378,6 @@ Example changes: - `change_column_default` - `create_table` / `drop_table` -**Note:** `with_lock_retries` method **cannot** be used with `disable_ddl_transaction!`. - **Note:** `with_lock_retries` method **cannot** be used within the `change` method, you must manually define the `up` and `down` methods to make the migration reversible. ### How the helper method works @@ -506,34 +552,12 @@ You can read more about adding [foreign key constraints to an existing column](d ## Adding Columns With Default Values -When adding columns with default values to non-empty tables, you must use -`add_column_with_default`. This method ensures the table is updated without -requiring downtime. This method is not reversible so you must manually define -the `up` and `down` methods in your migration class. - -For example, to add the column `foo` to the `projects` table with a default -value of `10` you'd write the following: - -```ruby -class MyMigration < ActiveRecord::Migration[4.2] - include Gitlab::Database::MigrationHelpers - disable_ddl_transaction! - - def up - add_column_with_default(:projects, :foo, :integer, default: 10) - end - - def down - remove_column(:projects, :foo) - end -end -``` +With PostgreSQL 11 being the minimum version since GitLab 13.0, adding columns with default values has become much easier and +the standard `add_column` helper should be used in all cases. -Keep in mind that this operation can easily take 10-15 minutes to complete on -larger installations (e.g. GitLab.com). As a result, you should only add -default values if absolutely necessary. There is a RuboCop cop that will fail if -this method is used on some tables that are very large on GitLab.com, which -would cause other issues. +Before PostgreSQL 11, adding a column with a default was problematic as it would +have caused a full table rewrite. The corresponding helper `add_column_with_default` +has been deprecated and will be removed in a later release. ## Changing the column default @@ -574,8 +598,7 @@ without requiring `disable_ddl_transaction!`. ## Updating an existing column To update an existing column to a particular value, you can use -`update_column_in_batches` (`add_column_with_default` uses this internally to -fill in the default value). This will split the updates into batches, so we +`update_column_in_batches`. This will split the updates into batches, so we don't update too many rows at in a single statement. This updates the column `foo` in the `projects` table to 10, where `some_column` @@ -701,7 +724,7 @@ set the limit to 8-bytes. This will allow the column to hold a value up to Rails migration example: ```ruby -add_column_with_default(:projects, :foo, :integer, default: 10, limit: 8) +add_column(:projects, :foo, :integer, default: 10, limit: 8) ``` ## Timestamp column type diff --git a/doc/development/multi_version_compatibility.md b/doc/development/multi_version_compatibility.md new file mode 100644 index 00000000000..aedd5c1ffb7 --- /dev/null +++ b/doc/development/multi_version_compatibility.md @@ -0,0 +1,62 @@ +# Compatibility with multiple versions of the application running at the same time + +When adding or changing features, we must be aware that there may be multiple versions of the application running +at the same time and connected to the same PostgreSQL and Redis databases. This could happen during a rolling deploy +when the servers are updated one by one. + +During a rolling deploy, post-deployment DB migrations are run after all the servers have been updated. This means the +servers could be in these intermediate states: + +1. Old application code running with new DB migrations already executed +1. New application code running with new DB migrations but without new post-deployment DB migrations + +We must make sure that the application works properly in these states. + +For GitLab.com, we also run a set of canary servers which run a more recent version of the application. Users with +the canary cookie set would be handled by these servers. Some URL patterns may also be forced to the canary servers, +even without the cookie being set. This also means that some pages may match the pattern and get handled by canary servers, +but AJAX requests to URLs (like the GraphQL endpoint) won't match the pattern. + +With this canary setup, we'd be in this mixed-versions state for an extended period of time until canary is promoted to +production and post-deployment migrations run. + +## Examples of previous incidents + +### Some links to issues and MRs were broken + +When we moved MR routes, users on the new servers were redirected to the new URLs. When these users shared these new URLs in +Markdown (or anywhere else), they were broken links for users on the old servers. + +For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/118840). + +### Stale cache in issue or merge request descriptions and comments + +We bumped the Markdown cache version and found a bug when a user edited a description or comment which was generated from a different Markdown +cache version. The cached HTML wasn't generated properly after saving. In most cases, this wouldn't have happened because users would have +viewed the Markdown before clicking **Edit** and that would mean the Markdown cache is refreshed. But because we run mixed versions, this is +more likely to happen. Another user on a different version could view the same page and refresh the cache to the other version behind the scenes. + +For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/208255). + +### Project service templates incorrectly copied + +We changed the column which indicates whether a service is a template. When we create services, we copy attributes from the template +and set this column to `false`. The old servers were still updating the old column, but that was fine because we had a DB trigger +that updated the new column from the old one. For the new servers though, they were only updating the new column and that same trigger +was now working against us and setting it back to the wrong value. + +For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9176). + +### Sidebar wasn't loading for some users + +We changed the data type of one GraphQL field. When a user opened an issue page from the new servers and the GraphQL AJAX request went +to the old servers, a type mismatch happened, which resulted in a JavaScript error that prevented the sidebar from loading. + +For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/1772). + +### CI artifact uploads were failing + +We added a `NOT NULL` constraint to a column and marked it as a `NOT VALID` constraint so that it is not enforced on existing rows. +But even with that, this was still a problem because the old servers were still inserting new rows with null values. + +For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/1944). diff --git a/doc/development/namespaces_storage_statistics.md b/doc/development/namespaces_storage_statistics.md index f175739e55e..3065d4f84a2 100644 --- a/doc/development/namespaces_storage_statistics.md +++ b/doc/development/namespaces_storage_statistics.md @@ -38,7 +38,7 @@ alternative method. ### Attempt A: PostgreSQL materialized view -Model can be updated through a refresh strategy based on a project routes SQL and a [materialized view](https://www.postgresql.org/docs/9.6/rules-materializedviews.html): +Model can be updated through a refresh strategy based on a project routes SQL and a [materialized view](https://www.postgresql.org/docs/11/rules-materializedviews.html): ```sql SELECT split_part("rs".path, '/', 1) as root_path, @@ -111,11 +111,11 @@ Directly relate the root namespace to its child namespaces, so whenever a namespace is created without a parent, this one is tagged with the root namespace ID: -| id | root_id | parent_id -|:---|:--------|:---------- -| 1 | 1 | NULL -| 2 | 1 | 1 -| 3 | 1 | 2 +| ID | root ID | parent ID | +|:---|:--------|:----------| +| 1 | 1 | NULL | +| 2 | 1 | 1 | +| 3 | 1 | 2 | To aggregate the statistics inside a namespace we'd execute something like: diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md index 8d9f7ca5069..aa76a9fec07 100644 --- a/doc/development/new_fe_guide/development/accessibility.md +++ b/doc/development/new_fe_guide/development/accessibility.md @@ -6,13 +6,13 @@ Using semantic HTML plays a key role when it comes to accessibility. WAI-ARIA (the Accessible Rich Internet Applications specification) defines a way to make Web content and Web applications more accessible to people with disabilities. -> Note: It is [recommended][using-aria] to use semantic elements as the primary method to achieve accessibility rather than adding aria attributes. Adding aria attributes should be seen as a secondary method for creating accessible elements. +> Note: It is [recommended](https://www.w3.org/TR/using-aria/#notes2) to use semantic elements as the primary method to achieve accessibility rather than adding aria attributes. Adding aria attributes should be seen as a secondary method for creating accessible elements. ### Role The `role` attribute describes the role the element plays in the context of the document. -Check the list of WAI-ARIA roles [here][roles] +Check the list of WAI-ARIA roles [here](https://www.w3.org/TR/wai-aria-1.1/#landmark_roles) ## Icons @@ -36,20 +36,11 @@ In forms we should use the `for` attribute in the label statement: ## Testing -1. On MacOS you can use [VoiceOver][voice-over] by pressing `cmd+F5`. -1. On Windows you can use [Narrator][narrator] by pressing Windows logo key + Ctrl + Enter. +1. On MacOS you can use [VoiceOver](https://www.apple.com/accessibility/mac/vision/) by pressing `cmd+F5`. +1. On Windows you can use [Narrator](https://www.microsoft.com/en-us/accessibility/windows) by pressing Windows logo key + Ctrl + Enter. ## Online resources -- [Chrome Accessibility Developer Tools][dev-tools] for testing accessibility -- [Audit Rules Page][audit-rules] for best practices -- [Lighthouse Accessibility Score][lighthouse] for accessibility audits - -[using-aria]: https://www.w3.org/TR/using-aria/#notes2 -[dev-tools]: https://github.com/GoogleChrome/accessibility-developer-tools -[audit-rules]: https://github.com/GoogleChrome/accessibility-developer-tools/wiki/Audit-Rules -[aria-w3c]: https://www.w3.org/TR/wai-aria-1.1/ -[roles]: https://www.w3.org/TR/wai-aria-1.1/#landmark_roles -[voice-over]: https://www.apple.com/accessibility/mac/vision/ -[narrator]: https://www.microsoft.com/en-us/accessibility/windows -[lighthouse]: https://developers.google.com/web/tools/lighthouse/scoring#a11y +- [Chrome Accessibility Developer Tools](https://github.com/GoogleChrome/accessibility-developer-tools) for testing accessibility +- [Audit Rules Page](https://github.com/GoogleChrome/accessibility-developer-tools/wiki/Audit-Rules) for best practices +- [Lighthouse Accessibility Score](https://developers.google.com/web/tools/lighthouse/scoring#a11y) for accessibility audits diff --git a/doc/development/new_fe_guide/modules/dirty_submit.md b/doc/development/new_fe_guide/modules/dirty_submit.md index 30cf8017820..dd336ad3a90 100644 --- a/doc/development/new_fe_guide/modules/dirty_submit.md +++ b/doc/development/new_fe_guide/modules/dirty_submit.md @@ -13,7 +13,7 @@ within the GitLab project. ## Usage -```js +```javascript import dirtySubmitFactory from './dirty_submit/dirty_submit_form'; new DirtySubmitForm(document.querySelector('form')); diff --git a/doc/development/newlines_styleguide.md b/doc/development/newlines_styleguide.md index a13adc2f13e..e298ba1d935 100644 --- a/doc/development/newlines_styleguide.md +++ b/doc/development/newlines_styleguide.md @@ -1,4 +1,4 @@ -# Newlines styleguide +# Newlines style guide This style guide recommends best practices for newlines in Ruby code. diff --git a/doc/development/ordering_table_columns.md b/doc/development/ordering_table_columns.md index cbfd05e731d..b68602ea30d 100644 --- a/doc/development/ordering_table_columns.md +++ b/doc/development/ordering_table_columns.md @@ -77,64 +77,64 @@ always be at the end of a table. Let's use the `events` table as an example, which currently has the following layout: -| Column | Type | Size | -|:------------|:----------------------------|:---------| -| id | integer | 4 bytes | -| target_type | character varying | variable | -| target_id | integer | 4 bytes | -| title | character varying | variable | -| data | text | variable | -| project_id | integer | 4 bytes | -| created_at | timestamp without time zone | 8 bytes | -| updated_at | timestamp without time zone | 8 bytes | -| action | integer | 4 bytes | -| author_id | integer | 4 bytes | +| Column | Type | Size | +|:--------------|:----------------------------|:---------| +| `id` | integer | 4 bytes | +| `target_type` | character varying | variable | +| `target_id` | integer | 4 bytes | +| `title` | character varying | variable | +| `data` | text | variable | +| `project_id` | integer | 4 bytes | +| `created_at` | timestamp without time zone | 8 bytes | +| `updated_at` | timestamp without time zone | 8 bytes | +| `action` | integer | 4 bytes | +| `author_id` | integer | 4 bytes | After adding padding to align the columns this would translate to columns being divided into fixed size chunks as follows: -| Chunk Size | Columns | -|:-----------|:------------------| -| 8 bytes | id | -| variable | target_type | -| 8 bytes | target_id | -| variable | title | -| variable | data | -| 8 bytes | project_id | -| 8 bytes | created_at | -| 8 bytes | updated_at | -| 8 bytes | action, author_id | +| Chunk Size | Columns | +|:-----------|:----------------------| +| 8 bytes | `id` | +| variable | `target_type` | +| 8 bytes | `target_id` | +| variable | `title` | +| variable | `data` | +| 8 bytes | `project_id` | +| 8 bytes | `created_at` | +| 8 bytes | `updated_at` | +| 8 bytes | `action`, `author_id` | This means that excluding the variable sized data and tuple header, we need at least 8 * 6 = 48 bytes per row. We can optimise this by using the following column order instead: -| Column | Type | Size | -|:------------|:----------------------------|:---------| -| created_at | timestamp without time zone | 8 bytes | -| updated_at | timestamp without time zone | 8 bytes | -| id | integer | 4 bytes | -| target_id | integer | 4 bytes | -| project_id | integer | 4 bytes | -| action | integer | 4 bytes | -| author_id | integer | 4 bytes | -| target_type | character varying | variable | -| title | character varying | variable | -| data | text | variable | +| Column | Type | Size | +|:--------------|:----------------------------|:---------| +| `created_at` | timestamp without time zone | 8 bytes | +| `updated_at` | timestamp without time zone | 8 bytes | +| `id` | integer | 4 bytes | +| `target_id` | integer | 4 bytes | +| `project_id` | integer | 4 bytes | +| `action` | integer | 4 bytes | +| `author_id` | integer | 4 bytes | +| `target_type` | character varying | variable | +| `title` | character varying | variable | +| `data` | text | variable | This would produce the following chunks: -| Chunk Size | Columns | -|:-----------|:-------------------| -| 8 bytes | created_at | -| 8 bytes | updated_at | -| 8 bytes | id, target_id | -| 8 bytes | project_id, action | -| 8 bytes | author_id | -| variable | target_type | -| variable | title | -| variable | data | +| Chunk Size | Columns | +|:-----------|:-----------------------| +| 8 bytes | `created_at` | +| 8 bytes | `updated_at` | +| 8 bytes | `id`, `target_id` | +| 8 bytes | `project_id`, `action` | +| 8 bytes | `author_id` | +| variable | `target_type` | +| variable | `title` | +| variable | `data` | Here we only need 40 bytes per row excluding the variable sized data and 24-byte tuple header. 8 bytes being saved may not sound like much, but for tables as diff --git a/doc/development/packages.md b/doc/development/packages.md index 66b50ce12c8..11fc3faf4ab 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -74,7 +74,7 @@ It is using the top-level group or namespace as the defining portion of the name To avoid name conflict for instance-level endpoints you will need to define a package naming convention that gives a way to identify the project that the package belongs to. This generally involves using the project -id or full project path in the package name. See +ID or full project path in the package name. See [Conan's naming convention](../user/packages/conan_repository/index.md#package-recipe-naming-convention) as an example. For group and project-level endpoints, naming can be less constrained and it will be up to the group and project @@ -110,7 +110,7 @@ The way new package systems are integrated in GitLab is using an [MVC](https://a Required actions are all the additional requests that GitLab will need to handle so the corresponding package manager CLI can work properly. It could be a search feature or an endpoint providing meta information about a package. For example: - For NuGet, the search request was implemented during the first MVC iteration, to support Visual Studio. -- For NPM, there is a metadata endpoint used by `npm` to get the tarball url. +- For NPM, there is a metadata endpoint used by `npm` to get the tarball URL. For the first MVC iteration, it's recommended to stay at the project level of the [remote hierarchy](#remote-hierarchy). Other levels can be tackled with [future Merge Requests](#future-work). @@ -139,7 +139,7 @@ During this phase, the idea is to collect as much information as possible about - **Requests**: Which requests are needed to have a working MVC. Ideally, produce a list of all the requests needed for the MVC (including required actions). Further investigation could provide an example for each request with the request and the response bodies. -- **Upload**: Carefully analyse how the upload process works. This will probably be the most +- **Upload**: Carefully analyze how the upload process works. This will probably be the most complex request to implement. A detailed analysis is desired here as uploads can be encoded in different ways (body or multipart) and can even be in a totally different format (for example, a JSON structure where the package file is a Base64 value of @@ -159,7 +159,7 @@ During this phase, the idea is to collect as much information as possible about The analysis usually takes a full milestone to complete, though it's not impossible to start the implementation in the same milestone. -In particular, the upload request can have some [requirements in the GitLab Workhorse project](#file-uploads). This project has a different release cycle than the rails backend. It's **strongly** recommended that you open an issue there as soon as the upload request analysis is done. This way GitLab Worhorse is already ready when the upload request is implemented on the rails backend. +In particular, the upload request can have some [requirements in the GitLab Workhorse project](#file-uploads). This project has a different release cycle than the rails backend. It's **strongly** recommended that you open an issue there as soon as the upload request analysis is done. This way GitLab Workhorse is already ready when the upload request is implemented on the rails backend. ### Implementation @@ -196,8 +196,8 @@ information like the file `name`, `side`, `sha1`, etc. If there is specific data necessary to be stored for only one package system support, consider creating a separate metadata model. See `packages_maven_metadata` table -and `Packages::MavenMetadatum` model as an example for package specific data, and `packages_conan_file_metadata` table -and `Packages::ConanFileMetadatum` model as an example for package file specific data. +and `Packages::Maven::Metadatum` model as an example for package specific data, and `packages_conan_file_metadata` table +and `Packages::Conan::FileMetadatum` model as an example for package file specific data. If there is package specific behavior for a given package manager, add those methods to the metadata models and delegate from the package model. diff --git a/doc/development/performance.md b/doc/development/performance.md index 5068103ff16..2e73161a11f 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -46,7 +46,7 @@ GitLab provides built-in tools to help improve performance and availability: GitLab team members can use [GitLab.com's performance monitoring systems](https://about.gitlab.com/handbook/engineering/monitoring/) located at <https://dashboards.gitlab.net>, this requires you to log in using your `@gitlab.com` email address. Non-GitLab team-members are advised to set up their -own InfluxDB and Grafana stack. +own Prometheus and Grafana stack. ## Benchmarks @@ -109,7 +109,7 @@ In short: By collecting snapshots of process state at regular intervals, profiling allows you to see where time is spent in a process. The [StackProf](https://github.com/tmm1/stackprof) gem is included in GitLab's development environment, allowing you to investigate -the behaviour of suspect code in detail. +the behavior of suspect code in detail. It's important to note that profiling an application *alters its performance*, and will generally be done *in an unrepresentative environment*. In particular, diff --git a/doc/development/permissions.md b/doc/development/permissions.md index bca137337fc..0772389bf9e 100644 --- a/doc/development/permissions.md +++ b/doc/development/permissions.md @@ -9,9 +9,9 @@ anything that deals with permissions, all of them should be considered. Groups and projects can have the following visibility levels: -- public (20) - an entity is visible to everyone -- internal (10) - an entity is visible to logged in users -- private (0) - an entity is visible only to the approved members of the entity +- public (`20`) - an entity is visible to everyone +- internal (`10`) - an entity is visible to logged in users +- private (`0`) - an entity is visible only to the approved members of the entity The visibility level of a group can be changed only if all subgroups and subprojects have the same or lower visibility level. (e.g., a group can be set diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 9ba6dfc110a..39ca846c1cc 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -19,6 +19,9 @@ The current stages are: <https://gitlab.com/gitlab-org/gitlab-foss>. - `prepare`: This stage includes jobs that prepare artifacts that are needed by jobs in subsequent stages. +- `build-images`: This stage includes jobs that prepare docker images + that are needed by jobs in subsequent stages or downstream pipelines. +- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests. - `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs. - `post-test`: This stage includes jobs that build reports or gather data from the `test` stage's jobs (e.g. coverage, Knapsack metadata etc.). @@ -30,7 +33,6 @@ The current stages are: that is deployed in the previous stage. - `post-qa`: This stage includes jobs that build reports or gather data from the `qa` stage's jobs (e.g. Review App performance report). -- `notification`: This stage includes jobs that sends notifications about pipeline status. - `pages`: This stage includes a job that deploys the various reports as GitLab Pages (e.g. <https://gitlab-org.gitlab.io/gitlab/coverage-ruby/>, <https://gitlab-org.gitlab.io/gitlab/coverage-javascript/>, @@ -68,12 +70,9 @@ that are scoped to a single [configuration parameter](../ci/yaml/README.md#confi | `.default-retry` | Allows a job to [retry](../ci/yaml/README.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. | | `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (e.g. tests). | | `.default-cache` | Allows a job to use a default `cache` definition suitable for Ruby/Rails and frontend tasks. | -| `.use-pg9` | Allows a job to use the `postgres:9.6.17` and `redis:alpine` services. | -| `.use-pg10` | Allows a job to use the `postgres:10.12` and `redis:alpine` services. | | `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. | -| `.use-pg9-ee` | Same as `.use-pg9` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. | -| `.use-pg10-ee` | Same as `.use-pg10` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. | | `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. | +| `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. | | `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. | ## `workflow:rules` @@ -129,11 +128,12 @@ and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anch | `changes:` patterns | Description | |------------------------------|--------------------------------------------------------------------------| +| `ci-patterns` | Only create job for CI config-related changes. | | `yaml-patterns` | Only create job for YAML-related changes. | | `docs-patterns` | Only create job for docs-related changes. | -| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. | +| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. | | `frontend-patterns` | Only create job for frontend-related changes. | -| `backstage-patterns` | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). | +| `backstage-patterns` | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). | | `code-patterns` | Only create job for code-related changes. | | `qa-patterns` | Only create job for QA-related changes. | | `code-backstage-patterns` | Combination of `code-patterns` and `backstage-patterns`. | @@ -151,112 +151,360 @@ request, be sure to start the `dont-interrupt-me` job before pushing. ## PostgreSQL versions testing +### Current versions testing + +| Where? | PostgreSQL version | +| ------ | ------ | +| MRs | 11 | +| `master` (non-scheduled pipelines) | 11 | +| 2-hourly scheduled pipelines | 11 | + +### Long-term plan + We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html): -| | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) | +| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) | | ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- | -| PG9.6 | nightly | - | - | - | - | - | - | - | -| PG10 | `master` | - | - | - | - | - | - | - | -| PG11 | MRs/`master` | MRs/`master` | MRs/`master` | MRs/`master` | MRs/`master` | MRs/`master` | nightly | - | -| PG12 | - | - | - | - | `master` | `master` | MRs/`master` | `master` | -| PG13 | - | - | - | - | - | - | - | MRs/`master` | +| PG9.6 | MRs/`master`/`2-hour`/`nightly` | - | - | - | - | - | - | - | +| PG10 | `nightly` | - | - | - | - | - | - | - | +| PG11 | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - | +| PG12 | - | - | - | - | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | `master`/`2-hour` | +| PG13 | - | - | - | - | - | - | - | MRs/`master`/`2-hour`/`nightly` | + +## Pipeline types + +Since we use the [`rules:`](../ci/yaml/README.md#rules) and [`needs:`](../ci/yaml/README.md#needs) keywords extensively, +we have four main pipeline types which are described below. Note that an MR that includes multiple types of changes would +have a pipelines that include jobs from multiple types (e.g. a combination of docs-only and code-only pipelines). + +### Docs-only MR pipeline + +Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/135236627> + +```mermaid +graph LR + subgraph "No needed jobs"; + 1-1["danger-review (3.5 minutes)"]; + click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" + 1-50["docs lint (6.75 minutes)"]; + click 1-50 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356757&udv=0" + end +``` + +### Code-only MR pipeline + +Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/136295694> + +```mermaid +graph RL; + classDef criticalPath fill:#f66; + + subgraph "No needed jobs"; + 1-1["danger-review (3.5 minutes)"]; + click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" + 1-2["build-qa-image (3.4 minutes)"]; + click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" + 1-3["compile-assets pull-cache (9.06 minutes)"]; + click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" + 1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"]; + click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" + 1-5["gitlab:assets:compile pull-cache (22 minutes)"]; + click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" + 1-6["setup-test-env (8.22 minutes)"]; + click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" + 1-7["review-stop-failed-deployment"]; + 1-8["dependency_scanning"]; + 1-9["qa:internal, qa:internal-as-if-foss"]; + 1-11["qa:selectors, qa:selectors-as-if-foss"]; + 1-14["retrieve-tests-metadata (1.5 minutes)"]; + click 1-14 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356697&udv=0" + 1-15["code_quality"]; + 1-16["brakeman-sast"]; + 1-17["eslint-sast"]; + 1-18["kubesec-sast"]; + 1-19["nodejs-scan-sast"]; + 1-20["secrets-sast"]; + + class 1-3 criticalPath; + class 1-6 criticalPath; + end + + 2_1-1["graphql-reference-verify (5 minutes)"]; + click 2_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356715&udv=0" + 2_1-2["memory-static (4.75 minutes)"]; + click 2_1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356721&udv=0" + 2_1-3["run-dev-fixtures (5 minutes)"]; + click 2_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356729&udv=0" + 2_1-4["run-dev-fixtures-ee (5 minutes)"]; + click 2_1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356731&udv=0" + subgraph "Needs `setup-test-env`"; + 2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6; + end + + 2_2-1["static-analysis (17 minutes)"]; + click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" + 2_2-2["frontend-fixtures (17.2 minutes)"]; + class 2_2-2 criticalPath; + click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0" + 2_2-3["frontend-fixtures-as-if-foss (8.75 minutes)"]; + click 2_2-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910154&udv=0" + 2_2-4["memory-on-boot (7.19 minutes)"]; + click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0" + 2_2-5["webpack-dev-server (6.1 minutes)"]; + click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0" + subgraph "Needs `setup-test-env` & `compile-assets`"; + 2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; + 2_2-3 --> 1-6 & 1-4; + end + + 2_3-1["build-assets-image (2.5 minutes)"]; + subgraph "Needs `gitlab:assets:compile`"; + 2_3-1 --> 1-5 + end + + 2_4-1["package-and-qa (manual)"]; + subgraph "Needs `build-qa-image`"; + 2_4-1 --> 1-2; + click 2_4-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914305&udv=0" + end + + 2_5-1["rspec & db jobs (12-22 minutes)"]; + subgraph "Needs `compile-assets`, `setup-test-env`, & `retrieve-tests-metadata`"; + 2_5-1 --> 1-3 & 1-6 & 1-14; + class 2_5-1 criticalPath; + click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations" + end + + 3_1-1["jest (15 minutes)"]; + class 3_1-1 criticalPath; + click 3_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914204&udv=0" + 3_1-2["karma (8 minutes)"]; + click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914200&udv=0" + 3_1-3["jest-as-if-foss (19.7 minutes)"]; + click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914205&udv=0" + 3_1-4["karma-as-if-foss (7.5 minutes)"]; + click 3_1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914203&udv=0" + subgraph "Needs `frontend-fixtures`"; + 3_1-1 & 3_1-2 --> 2_2-2; + 3_1-3 & 3_1-4 --> 2_2-3; + end + + 3_2-1["rspec:coverage (6.5 minutes)"]; + subgraph "Depends on `rspec` jobs"; + 3_2-1 -.->|"(don't use needs because of limitations)"| 2_5-1; + class 3_2-1 criticalPath; + click 3_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7248745&udv=0" + end + + 4_1-1["coverage-frontend (3.6 minutes)"]; + subgraph "Needs `jest`"; + 4_1-1 --> 3_1-1; + class 4_1-1 criticalPath; + click 4_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910777&udv=0" + end +``` + +### Frontend-only MR pipeline + +Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134661039> + +```mermaid +graph RL; + classDef criticalPath fill:#f66; + + subgraph "No needed jobs"; + 1-1["danger-review (3.5 minutes)"]; + click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" + 1-2["build-qa-image (3.4 minutes)"]; + click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" + 1-3["compile-assets pull-cache (9.06 minutes)"]; + click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" + 1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"]; + click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" + 1-5["gitlab:assets:compile pull-cache (22 minutes)"]; + click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" + 1-6["setup-test-env (8.22 minutes)"]; + click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" + 1-7["review-stop-failed-deployment"]; + 1-8["dependency_scanning"]; + 1-9["qa:internal, qa:internal-as-if-foss"]; + 1-11["qa:selectors, qa:selectors-as-if-foss"]; + 1-14["retrieve-tests-metadata (1.5 minutes)"]; + click 1-14 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356697&udv=0" + 1-15["code_quality"]; + 1-16["brakeman-sast"]; + 1-17["eslint-sast"]; + 1-18["kubesec-sast"]; + 1-19["nodejs-scan-sast"]; + 1-20["secrets-sast"]; + + class 1-3 criticalPath; + class 1-5 criticalPath; + class 1-6 criticalPath; + end + + 2_1-1["graphql-reference-verify (5 minutes)"]; + click 2_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356715&udv=0" + 2_1-2["memory-static (4.75 minutes)"]; + click 2_1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356721&udv=0" + 2_1-3["run-dev-fixtures (5 minutes)"]; + click 2_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356729&udv=0" + 2_1-4["run-dev-fixtures-ee (5 minutes)"]; + click 2_1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356731&udv=0" + subgraph "Needs `setup-test-env`"; + 2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6; + end + + 2_2-1["static-analysis (17 minutes)"]; + click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" + 2_2-2["frontend-fixtures (17.2 minutes)"]; + class 2_2-2 criticalPath; + click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0" + 2_2-3["frontend-fixtures-as-if-foss (8.75 minutes)"]; + click 2_2-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910154&udv=0" + 2_2-4["memory-on-boot (7.19 minutes)"]; + click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0" + 2_2-5["webpack-dev-server (6.1 minutes)"]; + click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0" + subgraph "Needs `setup-test-env` & `compile-assets`"; + 2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; + 2_2-3 --> 1-6 & 1-4; + end + + 2_3-1["build-assets-image (2.5 minutes)"]; + class 2_3-1 criticalPath; + subgraph "Needs `gitlab:assets:compile`"; + 2_3-1 --> 1-5 + end -## Directed acyclic graph + 2_4-1["package-and-qa (manual)"]; + subgraph "Needs `build-qa-image` & `build-assets-image`"; + 2_4-1 --> 1-2 & 2_3-1; + click 2_4-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914305&udv=0" + end + + 2_5-1["rspec & db jobs (12-22 minutes)"]; + subgraph "Needs `compile-assets`, `setup-test-env, & `retrieve-tests-metadata`"; + 2_5-1 --> 1-3 & 1-6 & 1-14; + class 2_5-1 criticalPath; + click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations" + end + + 2_6-1["review-build-cng (27.3 minutes)"]; + subgraph "Needs `build-assets-image`"; + 2_6-1 --> 2_3-1; + class 2_6-1 criticalPath; + click 2_6-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914314&udv=0" + end + + 3_1-1["jest (15 minutes)"]; + class 3_1-1 criticalPath; + click 3_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914204&udv=0" + 3_1-2["karma (8 minutes)"]; + click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914200&udv=0" + 3_1-3["jest-as-if-foss (19.7 minutes)"]; + click 3_1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914205&udv=0" + 3_1-4["karma-as-if-foss (7.5 minutes)"]; + click 3_1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914203&udv=0" + subgraph "Needs `frontend-fixtures`"; + 3_1-1 & 3_1-3 --> 2_2-2; + 3_1-2 & 3_1-4 --> 2_2-3; + end + + 3_2-1["rspec:coverage (6.5 minutes)"]; + subgraph "Depends on `rspec` jobs"; + 3_2-1 -.->|"(don't use needs because of limitations)"| 2_5-1; + class 3_2-1 criticalPath; + click 3_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7248745&udv=0" + end -We're using the [`needs:`](../ci/yaml/README.md#needs) keyword to -execute jobs out of order for the following jobs: + 4_1-1["coverage-frontend (3.6 minutes)"]; + subgraph "Needs `jest`"; + 4_1-1 --> 3_1-1; + class 4_1-1 criticalPath; + click 4_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910777&udv=0" + end + + 3_3-1["review-deploy (6 minutes)"]; + subgraph "Played by `review-build-cng`"; + 3_3-1 --> 2_6-1; + class 3_3-1 criticalPath; + click 3_3-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6721130&udv=0" + end + + 4_2-1["review-qa-smoke (8 minutes)"]; + click 4_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6729805&udv=0" + 4_2-2["review-performance (4 minutes)"]; + click 4_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356817&udv=0" + 4_2-3["dast (18 minutes)"]; + click 4_2-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356819&udv=0" + class 4_2-3 criticalPath; + subgraph "Played by `review-deploy`"; + 4_2-1 & 4_2-2 & 4_2-3 -.->|"(don't use needs because of limitations)"| 3_3-1; + end +``` + +### QA-only MR pipeline + +Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134645109> ```mermaid graph RL; - A[setup-test-env]; - B["gitlab:assets:compile pull-push-cache<br/>(canonical master only)"]; - C["gitlab:assets:compile pull-cache<br/>(canonical default refs only)"]; - D["cache gems<br/>(master and tags only)"]; - E[review-build-cng]; - F[build-qa-image]; - G[review-deploy]; - I["karma, jest"]; - I2["karma-as-if-foss, jest-as-if-foss<br/>(EE default refs only)"]; - J["compile-assets pull-push-cache<br/>(master only)"]; - J2["compile-assets pull-push-cache as-if-foss<br/>(EE master only)"]; - K[compile-assets pull-cache]; - K2["compile-assets pull-cache as-if-foss<br/>(EE default refs only)"]; - U[frontend-fixtures]; - U2["frontend-fixtures-as-if-foss<br/>(EE default refs only)"]; - V["webpack-dev-server, static-analysis"]; - M[coverage]; - O[coverage-frontend]; - N["pages (master only)"]; - Q[package-and-qa]; - S["RSpec<br/>(e.g. rspec unit pg10)"] - T[retrieve-tests-metadata]; - QA["qa:internal, qa:selectors"]; - QA2["qa:internal-as-if-foss, qa:selectors-as-if-foss<br/>(EE default refs only)"]; - X["docs lint, code_quality, sast, dependency_scanning, danger-review"]; - -subgraph "`prepare` stage" - A - B - C - F - K - K2 - J - J2 - T - end - -subgraph "`fixture` stage" - U -.-> |needs and depends on| A; - U -.-> |needs and depends on| K; - U2 -.-> |needs and depends on| A; - U2 -.-> |needs and depends on| K2; - end - -subgraph "`test` stage" - D -.-> |needs| A; - I -.-> |needs and depends on| U; - I2 -.-> |needs and depends on| U2; - L -.-> |needs and depends on| A; - S -.-> |needs and depends on| A; - S -.-> |needs and depends on| K; - S -.-> |needs and depends on| T; - L["db:*, gitlab:setup, graphql-docs-verify, downtime_check"] -.-> |needs| A; - V -.-> |needs and depends on| K; - X -.-> |needs| T; - QA -.-> |needs| T; - QA2 -.-> |needs| T; - end - -subgraph "`post-test` stage" - M --> |happens after| S - O --> |needs `jest`| I - end - -subgraph "`review-prepare` stage" - E -.-> |needs| C; - end - -subgraph "`review` stage" - G -.-> |needs| E - end - -subgraph "`qa` stage" - Q -.-> |needs| C; - Q -.-> |needs| F; - QA1["review-qa-smoke, review-qa-all, review-performance, dast"] -.-> |needs| G; - end - -subgraph "`post-qa` stage" - PQA1["parallel-spec-reports"] -.-> |depends on `review-qa-all`| QA1; + classDef criticalPath fill:#f66; + + subgraph "No needed jobs"; + 1-1["danger-review (3.5 minutes)"]; + click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" + 1-2["build-qa-image (3.4 minutes)"]; + click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" + 1-3["compile-assets pull-cache (9.06 minutes)"]; + click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" + 1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"]; + click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" + 1-5["gitlab:assets:compile pull-cache (22 minutes)"]; + click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" + 1-6["setup-test-env (8.22 minutes)"]; + click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" + 1-7["review-stop-failed-deployment"]; + 1-8["dependency_scanning"]; + 1-9["qa:internal, qa:internal-as-if-foss"]; + 1-11["qa:selectors, qa:selectors-as-if-foss"]; + 1-14["retrieve-tests-metadata (1.5 minutes)"]; + click 1-14 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356697&udv=0" + 1-15["code_quality"]; + 1-16["brakeman-sast"]; + 1-17["eslint-sast"]; + 1-18["kubesec-sast"]; + 1-19["nodejs-scan-sast"]; + 1-20["secrets-sast"]; + + class 1-5 criticalPath; + end + + 2_1-1["graphql-reference-verify (5 minutes)"]; + click 2_1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356715&udv=0" + subgraph "Needs `setup-test-env`"; + 2_1-1 --> 1-6; + end + + 2_2-1["static-analysis (17 minutes)"]; + click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" + subgraph "Needs `setup-test-env` & `compile-assets`"; + 2_2-1 --> 1-6 & 1-3; + end + + 2_3-1["build-assets-image (2.5 minutes)"]; + subgraph "Needs `gitlab:assets:compile`"; + 2_3-1 --> 1-5 + class 2_3-1 criticalPath; end -subgraph "`pages` stage" - N -.-> |depends on| C; - N -.-> |depends on karma| I; - N -.-> |depends on| M; - N --> |happens after| PQA1 - end + 2_4-1["package-and-qa (108 minutes)"]; + subgraph "Needs `build-qa-image` & `build-assets-image`"; + 2_4-1 --> 1-2 & 2_3-1; + class 2_4-1 criticalPath; + click 2_4-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914305&udv=0" + end ``` ## Test jobs @@ -268,6 +516,87 @@ for more information. Consult the [Review Apps](testing_guide/review_apps.md) dedicated page for more information. +## As-if-FOSS jobs + +The `* as-if-foss` jobs allows to run GitLab's test suite "as-if-FOSS", meaning as if the jobs would run in the context +of the `gitlab-org/gitlab-foss` project. These jobs are only created in the following cases: + +- `master` commits (pushes and scheduled pipelines). +- `gitlab-org/security/gitlab` merge requests. +- Merge requests which include `RUN AS-IF-FOSS` in their title. +- Merge requests that changes the CI config. + +The `* as-if-foss` jobs have the `FOSS_ONLY='1'` variable set and gets their EE-specific +folders removed before the tests start running. + +The intent is to ensure that a change won't introduce a failure once the `gitlab-org/gitlab` project will be synced to +the `gitlab-org/gitlab-foss` project. + +## Pre-clone step + +The `gitlab-org/gitlab` project on GitLab.com uses a [pre-clone step](https://gitlab.com/gitlab-org/gitlab/issues/39134) +to seed the project with a recent archive of the repository. This is done for +several reasons: + +- It speeds up builds because a 800 MB download only takes seconds, as opposed to a full Git clone. +- It significantly reduces load on the file server, as smaller deltas mean less time spent in `git pack-objects`. + +The pre-clone step works by using the `CI_PRE_CLONE_SCRIPT` variable +[defined by GitLab.com shared runners](../user/gitlab_com/index.md#pre-clone-script). + +The `CI_PRE_CLONE_SCRIPT` is currently defined as a project CI/CD +variable: + +```shell +echo "Downloading archived master..." +wget -O /tmp/gitlab.tar.gz https://storage.googleapis.com/gitlab-ci-git-repo-cache/project-278964/gitlab-master.tar.gz + +if [ ! -f /tmp/gitlab.tar.gz ]; then + echo "Repository cache not available, cloning a new directory..." + exit +fi + +rm -rf $CI_PROJECT_DIR +echo "Extracting tarball into $CI_PROJECT_DIR..." +mkdir -p $CI_PROJECT_DIR +cd $CI_PROJECT_DIR +tar xzf /tmp/gitlab.tar.gz +rm -f /tmp/gitlab.tar.gz +chmod a+w $CI_PROJECT_DIR +``` + +The first step of the script downloads `gitlab-master.tar.gz` from +Google Cloud Storage. There is a [GitLab CI job named `cache-repo`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/cache-repo.gitlab-ci.yml#L5) +that is responsible for keeping that archive up-to-date. Every two hours +on a scheduled pipeline, it does the following: + +1. Creates a fresh clone of the `gitlab-org/gitlab` repository on GitLab.com. +1. Saves the data as a `.tar.gz`. +1. Uploads it into the Google Cloud Storage bucket. + +When a CI job runs with this configuration, you'll see something like +this: + +```shell +$ eval "$CI_PRE_CLONE_SCRIPT" +Downloading archived master... +Extracting tarball into /builds/group/project... +Fetching changes... +Reinitialized existing Git repository in /builds/group/project/.git/ +``` + +Note that the `Reinitialized existing Git repository` message shows that +the pre-clone step worked. The runner runs `git init`, which +overwrites the Git configuration with the appropriate settings to fetch +from the GitLab repository. + +`CI_REPO_CACHE_CREDENTIALS` contains the Google Cloud service account +JSON for uploading to the `gitlab-ci-git-repo-cache` bucket. These +credentials are stored in the 1Password GitLab.com Production vault. + +Note that this bucket should be located in the same continent as the +runner, or [network egress charges will apply](https://cloud.google.com/storage/pricing). + --- [Return to Development documentation](README.md) diff --git a/doc/development/policies.md b/doc/development/policies.md index 4d045411156..62442de825a 100644 --- a/doc/development/policies.md +++ b/doc/development/policies.md @@ -95,7 +95,7 @@ Each line represents a rule that was evaluated. There are a few things to note: Here you can see that the first four rules were evaluated `false` for which user and subject. For example, you can see in the last line that -the rule was activated because the user `root` had Reporter access to +the rule was activated because the user `john` had Reporter access to `Project/4`. When a policy is asked whether a particular ability is allowed @@ -123,7 +123,7 @@ class FooPolicy < BasePolicy end ``` -Naively, if we call `Ability.can?(user1, :some_ability, foo)` and `Ability.can?(user2, :some_ability, foo)`, we would have to calculate the condition twice - since they are for different users. But if we use the `scope: :subject` option: +Naively, if we call `Ability.allowed?(user1, :some_ability, foo)` and `Ability.allowed?(user2, :some_ability, foo)`, we would have to calculate the condition twice - since they are for different users. But if we use the `scope: :subject` option: ```ruby condition(:expensive_condition, scope: :subject) { @subject.expensive_query? } diff --git a/doc/development/polymorphic_associations.md b/doc/development/polymorphic_associations.md index 5d69c8add38..b6567704d8e 100644 --- a/doc/development/polymorphic_associations.md +++ b/doc/development/polymorphic_associations.md @@ -4,7 +4,7 @@ Rails makes it possible to define so called "polymorphic associations". This usually works by adding two columns to a table: a target type column, and a -target id. For example, at the time of writing we have such a setup for +target ID. For example, at the time of writing we have such a setup for `members` with the following columns: - `source_type`: a string defining the model to use, can be either `Project` or diff --git a/doc/development/profiling.md b/doc/development/profiling.md index 04713055117..2589329fc83 100644 --- a/doc/development/profiling.md +++ b/doc/development/profiling.md @@ -87,7 +87,7 @@ printer.print(File.open('/tmp/profile.html', 'w')) [GitLab-Profiler](https://gitlab.com/gitlab-com/gitlab-profiler) is a project that builds on this to add some additional niceties, such as allowing -configuration with a single Yaml file for multiple URLs, and uploading of the +configuration with a single YAML file for multiple URLs, and uploading of the profile and log output to S3. For GitLab.com, you can find the latest results here (restricted to GitLab Team members only): diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md index f2c5b8b9848..b28fdb1252b 100644 --- a/doc/development/query_recorder.md +++ b/doc/development/query_recorder.md @@ -47,71 +47,71 @@ This could lead to false successes where subsequent "requests" could have querie There are multiple ways to find the source of queries. -1. The `QueryRecorder` `data` attribute stores queries by `file_name:line_number:method_name`. - Each entry is a `hash` with the following fields: - - - `count`: the number of times a query from this `file_name:line_number:method_name` was called - - `occurrences`: the actual `SQL` of each call - - `backtrace`: the stack trace of each call (if either of the two following options were enabled) - - `QueryRecorder#find_query` allows filtering queries by their `file_name:line_number:method_name` and - `count` attributes. For example: - - ```ruby - control = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page } - control.find_query(/.*note.rb.*/, 0, first_only: true) - ``` - - `QueryRecorder#occurrences_by_line_method` returns a sorted array based on `data`, sorted by `count`. - -1. You can output the call backtrace for the specific `QueryRecorder` instance you want - by using `ActiveRecord::QueryRecorder.new(query_recorder_debug: true)`. The output - will be in `test.log` - -1. Using the environment variable `QUERY_RECORDER_DEBUG`, the call backtrace will be output for all tests. - - To enable this, run the specs with the `QUERY_RECORDER_DEBUG` environment variable set. For example: - - ```shell - QUERY_RECORDER_DEBUG=1 bundle exec rspec spec/requests/api/projects_spec.rb - ``` - - This will log calls to QueryRecorder into the `test.log` file. For example: - - ```plaintext - QueryRecorder SQL: SELECT COUNT(*) FROM "issues" WHERE "issues"."deleted_at" IS NULL AND "issues"."project_id" = $1 AND ("issues"."state" IN ('opened')) AND "issues"."confidential" = $2 - --> /home/user/gitlab/gdk/gitlab/spec/support/query_recorder.rb:19:in `callback' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:127:in `finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `block in finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `each' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:36:in `finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:25:in `instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract_adapter.rb:478:in `log' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:601:in `exec_cache' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:585:in `execute_and_clear' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql/database_statements.rb:160:in `exec_query' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:356:in `select' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:32:in `select_all' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `block in select_all' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:83:in `cache_sql' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `select_all' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:270:in `execute_simple_calculation' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:227:in `perform_calculation' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:133:in `calculate' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:48:in `count' - --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:20:in `uncached_count' - --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `block in count' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `block in fetch' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:585:in `block in save_block_result_to_cache' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `block in instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications.rb:166:in `instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:584:in `save_block_result_to_cache' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `fetch' - --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `count' - --> /home/user/gitlab/gdk/gitlab/app/models/project.rb:1296:in `open_issues_count' - ``` +- Inspect the `QueryRecorder` `data` attribute. It stores queries by `file_name:line_number:method_name`. + Each entry is a `hash` with the following fields: + + - `count`: the number of times a query from this `file_name:line_number:method_name` was called + - `occurrences`: the actual `SQL` of each call + - `backtrace`: the stack trace of each call (if either of the two following options were enabled) + + `QueryRecorder#find_query` allows filtering queries by their `file_name:line_number:method_name` and + `count` attributes. For example: + + ```ruby + control = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page } + control.find_query(/.*note.rb.*/, 0, first_only: true) + ``` + + `QueryRecorder#occurrences_by_line_method` returns a sorted array based on `data`, sorted by `count`. + +- View the call backtrace for the specific `QueryRecorder` instance you want + by using `ActiveRecord::QueryRecorder.new(query_recorder_debug: true)`. The output + will be in `test.log` + +- Enable the call backtrace for all tests using the `QUERY_RECORDER_DEBUG` environment variable. + + To enable this, run the specs with the `QUERY_RECORDER_DEBUG` environment variable set. For example: + + ```shell + QUERY_RECORDER_DEBUG=1 bundle exec rspec spec/requests/api/projects_spec.rb + ``` + + This will log calls to QueryRecorder into the `test.log` file. For example: + + ```sql + QueryRecorder SQL: SELECT COUNT(*) FROM "issues" WHERE "issues"."deleted_at" IS NULL AND "issues"."project_id" = $1 AND ("issues"."state" IN ('opened')) AND "issues"."confidential" = $2 + --> /home/user/gitlab/gdk/gitlab/spec/support/query_recorder.rb:19:in `callback' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:127:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `block in finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `each' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:36:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:25:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract_adapter.rb:478:in `log' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:601:in `exec_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:585:in `execute_and_clear' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql/database_statements.rb:160:in `exec_query' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:356:in `select' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:32:in `select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `block in select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:83:in `cache_sql' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:270:in `execute_simple_calculation' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:227:in `perform_calculation' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:133:in `calculate' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:48:in `count' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:20:in `uncached_count' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `block in count' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `block in fetch' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:585:in `block in save_block_result_to_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `block in instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications.rb:166:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:584:in `save_block_result_to_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `fetch' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `count' + --> /home/user/gitlab/gdk/gitlab/app/models/project.rb:1296:in `open_issues_count' + ``` ## See also diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index 61d5277b1e6..96acce5e4df 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -1,8 +1,10 @@ # Rake tasks for developers -## Set up db with developer seeds +Rake tasks are available for developers and others contributing to GitLab. -Note that if your db user does not have advanced privileges you must create the db manually before running this command. +## Set up database with developer seeds + +Note that if your database user does not have advanced privileges, you must create the database manually before running this command. ```shell bundle exec rake setup @@ -12,13 +14,15 @@ The `setup` task is an alias for `gitlab:setup`. This tasks calls `db:reset` to create the database, and calls `db:seed_fu` to seed the database. Note: `db:setup` calls `db:seed` but this does nothing. -### Env variables +### Environment variables **MASS_INSERT**: Create millions of users (2m), projects (5m) and its relations. It's highly recommended to run the seed with it to catch slow queries while developing. Expect the process to take up to 20 extra minutes. -**LARGE_PROJECTS**: Create large projects (through import) from a predefined set of urls. +See also [Mass inserting Rails models](mass_insert.md). + +**LARGE_PROJECTS**: Create large projects (through import) from a predefined set of URLs. ### Seeding issues for all or a given project @@ -86,7 +90,7 @@ echo 'yes' | bundle exec rake setup To save you from answering `yes` manually. -### Discard stdout +### Discard `stdout` Since the script would print a lot of information, it could be slowing down your terminal, and it would generate more than 20G logs if you just redirect @@ -97,8 +101,8 @@ it to a file. If we don't care about the output, we could just redirect it to echo 'yes' | bundle exec rake setup > /dev/null ``` -Note that since you can't see the questions from stdout, you might just want -to `echo 'yes'` to keep it running. It would still print the errors on stderr +Note that since you can't see the questions from `stdout`, you might just want +to `echo 'yes'` to keep it running. It would still print the errors on `stderr` so no worries about missing errors. ### Extra Project seed options @@ -113,33 +117,35 @@ There are a few environment flags you can pass to change how projects are seeded In order to run the test you can use the following commands: -- `bin/rake spec` to run the rspec suite +- `bin/rake spec` to run the RSpec suite - `bin/rake spec:unit` to run only the unit tests - `bin/rake spec:integration` to run only the integration tests - `bin/rake spec:system` to run only the system tests - `bin/rake karma` to run the Karma test suite -Note: `bin/rake spec` takes significant time to pass. -Instead of running full test suite locally you can save a lot of time by running -a single test or directory related to your changes. After you submit merge request +`bin/rake spec` takes significant time to pass. +Instead of running the full test suite locally, you can save a lot of time by running +a single test or directory related to your changes. After you submit a merge request, CI will run full test suite for you. Green CI status in the merge request means full test suite is passed. -Note: You can't run `rspec .` since this will try to run all the `_spec.rb` +You can't run `rspec .` since this will try to run all the `_spec.rb` files it can find, also the ones in `/tmp` -Note: You can pass RSpec command line options to the `spec:unit`, -`spec:integration`, and `spec:system` tasks, e.g. `bin/rake "spec:unit[--tag ~geo --dry-run]"`. +You can pass RSpec command line options to the `spec:unit`, +`spec:integration`, and `spec:system` tasks. For example, `bin/rake "spec:unit[--tag ~geo --dry-run]"`. -To run a single test file you can use: +For an RSpec test, to run a single test file you can run: -- `bin/rspec spec/controllers/commit_controller_spec.rb` for a rspec test +```shell +bin/rspec spec/controllers/commit_controller_spec.rb +``` To run several tests inside one directory: -- `bin/rspec spec/requests/api/` for the rspec tests if you want to test API only +- `bin/rspec spec/requests/api/` for the RSpec tests if you want to test API only -### Speed-up tests, Rake tasks, and migrations +### Speed up tests, Rake tasks, and migrations [Spring](https://github.com/rails/spring) is a Rails application preloader. It speeds up development by keeping your application running in the background so @@ -172,18 +178,16 @@ This will compile and minify all JavaScript and CSS assets and copy them along with all other frontend assets (images, fonts, etc) into `/public/assets` where they can be easily inspected. -## Updating Emoji Aliases +## Emoji tasks -To update the Emoji aliases file (used for Emoji autocomplete) you must run the +To update the Emoji aliases file (used for Emoji autocomplete), run the following: ```shell bundle exec rake gemojione:aliases ``` -## Updating Emoji Digests - -To update the Emoji digests file (used for Emoji autocomplete) you must run the +To update the Emoji digests file (used for Emoji autocomplete), run the following: ```shell @@ -193,9 +197,7 @@ bundle exec rake gemojione:digests This will update the file `fixtures/emojis/digests.json` based on the currently available Emoji. -## Emoji Sprites - -Generating a sprite file containing all the Emoji can be done by running: +To generate a sprite file containing all the Emoji, run: ```shell bundle exec rake gemojione:sprite @@ -206,7 +208,7 @@ such changes, first generate the `emoji.png` spritesheet with the above Rake task, then check the dimensions of the new spritesheet and update the `SPRITESHEET_WIDTH` and `SPRITESHEET_HEIGHT` constants accordingly. -## Updating project templates +## Update project templates Starting a project from a template needs this project to be exported. On a up to date master branch run: @@ -249,7 +251,7 @@ bundle exec rake db:obsolete_ignored_columns Feel free to remove their definitions from their `ignored_columns` definitions. -## Update GraphQL Documentation and Schema definitions +## Update GraphQL documentation and schema definitions To generate GraphQL documentation based on the GitLab schema, run: diff --git a/doc/development/reactive_caching.md b/doc/development/reactive_caching.md index bc5fbb58af9..f3386305e93 100644 --- a/doc/development/reactive_caching.md +++ b/doc/development/reactive_caching.md @@ -85,6 +85,9 @@ The ReactiveCaching concern can be used in models as well as `project_services` 1. Implement the `calculate_reactive_cache` method in your model/service. 1. Call `with_reactive_cache` in your model/service where the cached value is needed. +1. If the `calculate_reactive_cache` method above submits requests to external services +(e.g. Prometheus, K8s), make sure to change the +[`reactive_cache_work_type` accordingly](#selfreactive_cache_work_type). ### In controllers @@ -244,6 +247,13 @@ and will silently raise `ReactiveCaching::ExceededReactiveCacheLimit` on Sentry. self.reactive_cache_hard_limit = 5.megabytes ``` +#### `self.reactive_cache_work_type` + +- This is the type of work performed by the `calculate_reactive_cache` method. Based on this attribute, +it's able to pick the right worker to process the caching job. Make sure to +set it as `:external_dependency` if the work performs any external request +(e.g. Kubernetes, Sentry). + #### `self.reactive_cache_worker_finder` - This is the method used by the background worker to find or generate the object on @@ -256,7 +266,7 @@ which `calculate_reactive_cache` can be called. end ``` -- The default behaviour can be overridden by defining a custom `reactive_cache_worker_finder`. +- The default behavior can be overridden by defining a custom `reactive_cache_worker_finder`. ```ruby class Foo < ApplicationRecord diff --git a/doc/development/repository_mirroring.md b/doc/development/repository_mirroring.md index 0a0c91821cf..1d4dbe88399 100644 --- a/doc/development/repository_mirroring.md +++ b/doc/development/repository_mirroring.md @@ -2,9 +2,10 @@ ## Deep Dive -In December 2018, Tiago Botelho hosted a [Deep Dive] on GitLab's [Pull Repository Mirroring functionality] to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube], and the slides in [PDF]. Everything covered in this deep dive was accurate as of GitLab 11.6, and while specific details may have changed since then, it should still serve as a good introduction. - -[Deep Dive]: https://gitlab.com/gitlab-org/create-stage/issues/1 -[Pull Repository Mirroring functionality]: ../user/project/repository/repository_mirroring.md#pulling-from-a-remote-repository-starter -[recording on YouTube]: https://www.youtube.com/watch?v=sSZq0fpdY-Y -[PDF]: https://gitlab.com/gitlab-org/create-stage/uploads/8693404888a941fd851f8a8ecdec9675/Gitlab_Create_-_Pull_Mirroring_Deep_Dive.pdf +In December 2018, Tiago Botelho hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) +on GitLab's [Pull Repository Mirroring functionality](../user/project/repository/repository_mirroring.md#pulling-from-a-remote-repository-starter) +to share his domain specific knowledge with anyone who may work in this part of the +code base in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=sSZq0fpdY-Y), +and the slides in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/8693404888a941fd851f8a8ecdec9675/Gitlab_Create_-_Pull_Mirroring_Deep_Dive.pdf). +Everything covered in this deep dive was accurate as of GitLab 11.6, and while specific +details may have changed since then, it should still serve as a good introduction. diff --git a/doc/development/scalability.md b/doc/development/scalability.md index 9607bce3bc4..ba25e169d66 100644 --- a/doc/development/scalability.md +++ b/doc/development/scalability.md @@ -214,7 +214,7 @@ Redis process. Single-core: Like PgBouncer, a single Redis process can only use one core. It does not support multi-threading. -Dumb secondaries: Redis secondaries (aka slaves) don't actually +Dumb secondaries: Redis secondaries (aka replicas) don't actually handle any load. Unlike PostgreSQL secondaries, they don't even serve read queries. They simply replicate data from the primary and take over only when the primary fails. @@ -231,8 +231,8 @@ election to determine a new leader. No leader: A Redis cluster can get into a mode where there are no primaries. For example, this can happen if Redis nodes are misconfigured to follow the wrong node. Sometimes this requires forcing one node to -become a primary via the [`SLAVEOF NO ONE` -command](https://redis.io/commands/slaveof). +become a primary via the [`REPLICAOF NO ONE` +command](https://redis.io/commands/replicaof). ### Sidekiq diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 0367db8939a..b473c310647 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -1,4 +1,4 @@ -# Security Guidelines +# Secure Coding Guidelines This document contains descriptions and guidelines for addressing security vulnerabilities commonly identified in the GitLab codebase. They are intended @@ -26,7 +26,7 @@ Improper permission handling can have significant impacts on the security of an Some situations may reveal [sensitive data](https://gitlab.com/gitlab-com/gl-infra/production/issues/477) or allow a malicious actor to perform [harmful actions](https://gitlab.com/gitlab-org/gitlab/issues/8180). The overall impact depends heavily on what resources can be accessed or modified improperly. -A common vulnerability when permission checks are missing is called [IDOR](https://www.owasp.org/index.php/Testing_for_Insecure_Direct_Object_References_(OTG-AUTHZ-004)) for Insecure Direct Object References. +A common vulnerability when permission checks are missing is called [IDOR](https://owasp.org/www-project-web-security-testing-guide/latest/4-Web_Application_Security_Testing/05-Authorization_Testing/04-Testing_for_Insecure_Direct_Object_References) for Insecure Direct Object References. ### When to Consider @@ -49,8 +49,8 @@ Be careful to **also test [visibility levels](https://gitlab.com/gitlab-org/gitl Some example of well implemented access controls and tests: 1. [example1](https://dev.gitlab.org/gitlab/gitlab-ee/merge_requests/710/diffs?diff_id=13750#af40ef0eaae3c1e018809e1d88086e32bccaca40_43_43) -1. [example2](https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/2511/diffs#ed3aaab1510f43b032ce345909a887e5b167e196_142_155) -1. [example3](https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/3170/diffs?diff_id=17494) +1. [example2](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/2511/diffs#ed3aaab1510f43b032ce345909a887e5b167e196_142_155) +1. [example3](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/3170/diffs?diff_id=17494) **NB:** any input from development team is welcome, e.g. about rubocop rules. @@ -74,11 +74,11 @@ text = "foo\nbar" p text.match /^bar$/ ``` -The output of this example is `#<MatchData "bar">`, as Ruby treats the input `text` line by line. In order to match the whole __string__ the Regex anchors `\A` and `\z` should be used according to [Rubular](https://rubular.com/). +The output of this example is `#<MatchData "bar">`, as Ruby treats the input `text` line by line. In order to match the whole __string__ the Regex anchors `\A` and `\z` should be used. #### Impact -This Ruby Regex speciality can have security impact, as often regular expressions are used for validations or to impose restrictions on user-input. +This Ruby Regex specialty can have security impact, as often regular expressions are used for validations or to impose restrictions on user-input. #### Examples @@ -104,15 +104,64 @@ Here `params[:ip]` should not contain anything else but numbers and dots. Howeve In most cases the anchors `\A` for beginning of text and `\z` for end of text should be used instead of `^` and `$`. -### Further Links +## Denial of Service (ReDoS) + +[ReDoS](https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS) is a possible attack if the attacker knows +or controls the regular expression (regex) used, and is able to enter user input to match against the bad regular expression. + +### Impact + +The resource, for example Unicorn, Puma, or Sidekiq, can be made to hang as it takes a long time to evaulate the bad regex match. + +### Examples + +GitLab-specific examples can be found in the following merge requests: + +- [MR25314](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25314) +- [MR25122](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25122#note_289087459) + +Consider the following example application, which defines a check using a regular expression. A user entering `user@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!.com` as the email on a form will hang the web server. + +```ruby +class Email < ApplicationRecord + DOMAIN_MATCH = Regexp.new('([a-zA-Z0-9]+)+\.com') + + validates :domain_matches + + private + + def domain_matches + errors.add(:email, 'does not match') if email =~ DOMAIN_MATCH + end +``` + +### Mitigation + +GitLab has `Gitlab::UntrustedRegexp` which internally uses the [`re2`](https://github.com/google/re2/wiki/Syntax) library. +By utilizing `re2`, we get a strict limit on total execution time, and a smaller subset of available regex features. + +All user-provided regexes should use `Gitlab::UntrustedRegexp`. + +For other regexes, here are a few guidelines: + +- Remove unnecessary backtracking. +- Avoid nested quantifiers if possible. +- Try to be as precise as possible in your regex and avoid the `.` if something else can be used (e.g.: Use `_[^_]+_` instead of `_.*_` to match `_text here_`). + +An example can be found [in this commit](https://gitlab.com/gitlab-org/gitlab/commit/717824144f8181bef524592eab882dd7525a60ef). + +## Further Links - [Rubular](https://rubular.com/) is a nice online tool to fiddle with Ruby Regexps. +- [Runaway Regular Expressions](https://www.regular-expressions.info/catastrophic.html) +- [The impact of regular expression denial of service (ReDoS) in practice: an empirical study at the ecosystem scale](http://people.cs.vt.edu/~davisjam/downloads/publications/DavisCoghlanServantLee-EcosystemREDOS-ESECFSE18.pdf). This research paper discusses approaches to automatically detect ReDoS vulnerabilities. +- [Freezing the web: A study of redos vulnerabilities in JavaScript-based web servers](https://www.usenix.org/system/files/conference/usenixsecurity18/sec18-staicu.pdf). Another research paper about detecting ReDoS vulnerabilities. ## Server Side Request Forgery (SSRF) ### Description -A [Server-side Request Forgery (SSRF)][1] is an attack in which an attacker +A [Server-side Request Forgery (SSRF)](https://www.hackerone.com/blog-How-To-Server-Side-Request-Forgery-SSRF) is an attack in which an attacker is able coerce a application into making an outbound request to an unintended resource. This resource is usually internal. In GitLab, the connection most commonly uses HTTP, but an SSRF can be performed with any protocol, such as @@ -122,8 +171,6 @@ With an SSRF attack, the UI may or may not show the response. The latter is called a Blind SSRF. While the impact is reduced, it can still be useful for attackers, especially for mapping internal network services as part of recon. -[1]: https://www.hackerone.com/blog-How-To-Server-Side-Request-Forgery-SSRF - ### Impact The impact of an SSRF can vary, depending on what the application server @@ -155,16 +202,14 @@ The preferred SSRF mitigations within GitLab are: #### GitLab HTTP Library -The [GitLab::HTTP][2] wrapper library has grown to include mitigations for all of the GitLab-known SSRF vectors. It is also configured to respect the +The [GitLab::HTTP](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/http.rb) wrapper library has grown to include mitigations for all of the GitLab-known SSRF vectors. It is also configured to respect the `Outbound requests` options that allow instance administrators to block all internal connections, or limit the networks to which connections can be made. In some cases, it has been possible to configure GitLab::HTTP as the HTTP connection library for 3rd-party gems. This is preferrable over re-implementing the mitigations for a new feature. -- [More details](https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/2530/diffs) - -[2]: https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/http.rb +- [More details](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/2530/diffs) #### Feature-specific Mitigations @@ -185,9 +230,7 @@ For situtions in which a whitelist or GitLab:HTTP cannot be used, it will be nec - For HTTP connections: Disable redirects or validate the redirect destination - To mitigate DNS rebinding attacks, validate and use the first IP address received -See [url_blocker_spec.rb][3] for examples of SSRF payloads - -[3]: https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/lib/gitlab/url_blocker_spec.rb +See [url_blocker_spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/lib/gitlab/url_blocker_spec.rb) for examples of SSRF payloads ## XSS guidelines @@ -236,7 +279,7 @@ For any and all input fields, ensure to define expectations on the type/format o - Validate the input using a [whitelist approach](https://youtu.be/2VFavqfDS6w?t=7816) to only allow characters through which you are expecting to receive for the field. - Input which fails validation should be **rejected**, and not sanitized. -Note that blacklists should be avoided, as it is near impossible to block all [variations of XSS](https://www.owasp.org/index.php/XSS_Filter_Evasion_Cheat_Sheet). +Note that blacklists should be avoided, as it is near impossible to block all [variations of XSS](https://owasp.org/www-community/xss-filter-evasion-cheatsheet). #### Output encoding diff --git a/doc/development/shell_scripting_guide/index.md b/doc/development/shell_scripting_guide/index.md index 387ef01bdcf..99cd1b9d67f 100644 --- a/doc/development/shell_scripting_guide/index.md +++ b/doc/development/shell_scripting_guide/index.md @@ -80,7 +80,20 @@ We format shell scripts according to the [Google Shell Style Guide](https://goog so the following `shfmt` invocation should be applied to the project's script files: ```shell -shfmt -i 2 -ci scripts/**/*.sh +shfmt -i 2 -ci -w scripts/**/*.sh +``` + +In addition to the [Linting](#linting) GitLab CI/CD job, all projects with shell scripts should also +use this job: + +```yaml +shfmt: + image: mvdan/shfmt:v3.1.0-alpine + stage: test + before_script: + - shfmt -version + script: + - shfmt -i 2 -ci -d scripts # path to your shell scripts ``` TIP: **Tip:** @@ -88,11 +101,6 @@ By default, shfmt will use the [shell detection](https://github.com/mvdan/sh#shf and ignore files starting with a period. To override this, use `-ln` flag to specify the shell dialect: `-ln posix` or `-ln bash`. -NOTE: **Note:** -Currently, the `shfmt` tool [is not shipped](https://github.com/mvdan/sh/issues/68) as a Docker image containing -a Linux shell. This makes it impossible to use the [official Docker image](https://hub.docker.com/r/mvdan/shfmt) -in GitLab Runner. This [may change](https://github.com/mvdan/sh/issues/68#issuecomment-507721371) in future. - ## Testing NOTE: **Note:** diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index 0ca25f43345..a5d0eecdc7b 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -82,6 +82,11 @@ As a general rule, a worker can be considered idempotent if: A good example of that would be a cache expiration worker. +NOTE: **Note:** +A job scheduled for an idempotent worker will automatically be +[deduplicated](#deduplication) when an unstarted job with the same +arguments is already in the queue. + ### Ensuring a worker is idempotent Make sure the worker tests pass using the following shared example: @@ -118,8 +123,50 @@ It's encouraged to only have the `idempotent!` call in the top-most worker class the `perform` method is defined in another class or module. NOTE: **Note:** -Note that a cop will fail if the worker class is not marked as idempotent. -Consider skipping the cop if you're not confident your job can safely run multiple times. +If the worker class is not marked as idempotent, a cop will fail. +Consider skipping the cop if you're not confident your job can safely +run multiple times. + +### Deduplication + +When a job for an idempotent worker is enqueued while another +unstarted job is already in the queue, GitLab drops the second +job. The work is skipped because the same work would be +done by the job that was scheduled first; by the time the second +job executed, the first job would do nothing. + +For example, `AuthorizedProjectsWorker` takes a user ID. When the +worker runs, it recalculates a user's authorizations. GitLab schedules +this job each time an action potentially changes a user's +authorizations. If the same user is added to two projects at the +same time, the second job can be skipped if the first job hasn't +begun, because when the first job runs, it creates the +authorizations for both projects. + +GitLab doesn't skip jobs scheduled in the future, as we assume that +the state will have changed by the time the job is scheduled to +execute. + +More [deduplication strategies have been suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/195). If you are implementing a worker that +could benefit from a different strategy, please comment in the issue. + +If the automatic deduplication were to cause issues in certain +queues. This can be temporarily disabled by enabling a feature flag +named `disable_<queue name>_deduplication`. For example to disable +deduplication for the `AuthorizedProjectsWorker`, we would enable the +feature flag `disable_authorized_projects_deduplication`. + +From chatops: + +```shell +/chatops run feature set disable_authorized_projects_deduplication true +``` + +From the rails console: + +```ruby +Feature.enable!(:disable_authorized_projects_deduplication) +``` ## Job urgency diff --git a/doc/development/telemetry/index.md b/doc/development/telemetry/index.md new file mode 100644 index 00000000000..32f63d5221e --- /dev/null +++ b/doc/development/telemetry/index.md @@ -0,0 +1,165 @@ +# Telemetry Guide + +At GitLab, we collect telemetry for the purpose of helping us build a better GitLab. Data about how GitLab is used is collected to better understand what parts of GitLab needs improvement and what features to build next. Telemetry also helps our team better understand the reasons why people use GitLab and with this knowledge we are able to make better product decisions. + +We also encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. By enabling tracking, users can: + +- Contribute back to the wider community. +- Help GitLab improve on the product. + +This documentation consists of three guides providing an overview of Telemetry at GitLab. + +Telemetry Guide: + + 1. [Our tracking tools](#our-tracking-tools) + 1. [What data can be tracked](#what-data-can-be-tracked) + 1. [Telemetry systems overview](#telemetry-systems-overview) + +[Usage Ping Guide](usage_ping.md) + + 1. [What is Usage Ping](usage_ping.md#what-is-usage-ping) + 1. [Usage Ping payload](usage_ping.md#usage-ping-payload) + 1. [Disabling Usage Ping](usage_ping.md#disabling-usage-ping) + 1. [Usage Ping request flow](usage_ping.md#usage-ping-request-flow) + 1. [How Usage Ping works](usage_ping.md#how-usage-ping-works) + 1. [Implementing Usage Ping](usage_ping.md#implementing-usage-ping) + 1. [Developing and testing usage ping](usage_ping.md#developing-and-testing-usage-ping) + +[Snowplow Guide](snowplow.md) + +1. [What is Snowplow](snowplow.md#what-is-snowplow) +1. [Snowplow schema](snowplow.md#snowplow-schema) +1. [Enabling Snowplow](snowplow.md#enabling-snowplow) +1. [Snowplow request flow](snowplow.md#snowplow-request-flow) +1. [Implementing Snowplow JS (Frontend) tracking](snowplow.md#implementing-snowplow-js-frontend-tracking) +1. [Implementing Snowplow Ruby (Backend) tracking](snowplow.md#implementing-snowplow-ruby-backend-tracking) +1. [Developing and testing Snowplow](snowplow.md#developing-and-testing-snowplow) + +More useful links: + +- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#-data-analysis-process) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/) + +## Our tracking tools + +In this section we will explain the six different technologies we use to gather product usage data. + +**Snowplow JS (Frontend)** + +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) is a frontend tracker for client-side events. + +**Snowplow Ruby (Backend)** + +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) is a backend tracker for server-side events. + +**Usage Ping** + +Usage Ping is a method for GitLab Inc to collect usage data on a GitLab instance. Usage Ping is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. This high-level data is used to help our product, support, and sales teams. + +Read more about how this works in the [Usage Ping guide](usage_ping.md) + +**Database import** + +Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/#extract-and-load). + +**Log system** + +System logs are the application logs generated from running the GitLab Rails application. For more details, see the [log system](../../administration/logs.md) and [logging infrastructure](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#logging-infrastructure-overview). + +## What data can be tracked + +Our different tracking tools allows us to track different types of events. The event types and examples of what data can be tracked are outlined below. + +| Event Type | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Log system | +| ------ | ------ | ------ | ------ | ------ | ------ | +| Database counts | ❌ | ❌ | ✅ | ✅ | ❌ | +| Pageview events | ✅ | ✅ | ❌ | ❌ | ❌ | +| UI events | ✅ | ❌ | ❌ | ❌ | ❌ | +| CRUD and API events | ❌ | ✅ | ❌ | ❌ | ❌ | +| Event funnels | ✅ | ✅ | ❌ | ❌ | ❌ | +| PostgreSQL Data | ❌ | ❌ | ❌ | ✅ | ❌ | +| Logs | ❌ | ❌ | ❌ | ❌ | ✅ | +| External services | ❌ | ❌ | ❌ | ❌ | ❌ | + +**Database counts** + +- How many Projects have been created by unique users +- How many users logged in the past 28 day + +Database counts are row counts for different tables in an instance’s database. These are SQL count queries which have been filtered, grouped, or aggregated which provide high level usage data. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql) + +**Pageview events** + +- How many sessions visited the /dashboard/groups page + +**UI Events** + +- How many sessions clicked on a button or link +- How many sessions closed a modal + +UI events are any interface-driven actions from the browser including click data. + +**CRUD or API events** + +- How many Git pushes were made +- How many GraphQL queries were made +- How many requests were made to a Rails action or controller. + +These are backend events that include the creation, read, update, deletion of records and other events that might be triggered from layers that aren't necessarily only available in the interface. + +**Event funnels** + +- How many sessions performed action A, B, then C +- What is our conversion rate from step A to B? + +**PostgreSQL data** + +These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql) + +**Logs** + +These are raw logs such as the [Production logs](../../administration/logs.md#production_jsonlog), [API logs](../../administration/logs.md#api_jsonlog), or [Sidekiq logs](../../administration/logs.md#sidekiqlog). See the [overview of Logging Infrastructure](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#logging-infrastructure-overview) for more details. + +**External services** + +These are external services a GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. + +## Telemetry systems overview + +The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed nstances. + + + +[Source file](https://app.diagrams.net/#G13DVpN-XnhWGz9tqReIj8pp1UE4ehk_EC) + +### GitLab Inc + +For Telemetry purposes, GitLab Inc has three major components: + +1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://about.gitlab.com/handbook/engineering/infrastructure/library/snowplow/) and GitLab's Versions Application. +1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. +1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana + +### Self-managed + +For Telemetry purposes, self-managed instances have two major components: + +1. Data infrastructure: Having a data infrastructure setup is optional on self-managed instances. If you'd like to collect Snowplow tracking events for your self-managed instance, you can setup your own self-managed Snowplow collector and configure your Snowplow events to point to your own collector. +1. GitLab: A self-managed GitLab instance contains all of the same components as GitLab.com mentioned above. + +### Differences between GitLab Inc and Self-managed + +As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Ping, and PostgreSQL database imports all flow into GitLab Inc's data fnfrastructure. However, on self-managed, only Usage Ping flows into GitLab Inc's data infrastructure. + +As shown by the green lines, on GitLab.com system logs flow into GitLab Inc's monitoring infrastructure. On self-managed, there are no logs sent to GitLab Inc's monitoring infrastructure. + +The differences between GitLab.com and self-managed are summarized below: + +| Environment | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Logs system | +| ------ | ------ | ------ | ------ | ------ | ------ | +| GitLab.com | ✅ | ✅ | ✅ | ✅ | ✅ | +| Self-Managed | ❌(1) | ❌(1) | ✅ | ❌ | ❌ | + +Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md new file mode 100644 index 00000000000..aeaad6e5624 --- /dev/null +++ b/doc/development/telemetry/snowplow.md @@ -0,0 +1,393 @@ +# Snowplow Guide + +This guide provides a details about how Snowplow works. It includes the following sections: + +1. [What is Snowplow](#what-is-snowplow) +1. [Snowplow schema](#snowplow-schema) +1. [Enabling Snowplow](#enabling-snowplow) +1. [Snowplow request flow](#snowplow-request-flow) +1. [Implementing Snowplow JS (Frontend) tracking](#implementing-snowplow-js-frontend-tracking) +1. [Implementing Snowplow Ruby (Backend) tracking](#implementing-snowplow-ruby-backend-tracking) +1. [Developing and testing Snowplow](#developing-and-testing-snowplow) + +For more information about Telemetry, see: + +- [Telemetry Guide](index.md) +- [Usage Ping Guide](usage_ping.md) + +More useful links: + +- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#-data-analysis-process) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/) + +## What is Snowplow + +Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. + +From [Snowplow's documentation](https://github.com/snowplow/snowplow), Snowplow consists of six loosely-coupled sub-systems: + +- **Trackers** fire Snowplow events. Currently Snowplow has 12 trackers, covering web, mobile, desktop, server and IoT +- **Collectors** receive Snowplow events from trackers. Currently we have three different event collectors, sinking events either to Amazon S3, Apache Kafka or Amazon Kinesis +- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. Currently we have a Hadoop-based enrichment process, and a Kinesis- or Kafka-based process +- **Storage** is where the Snowplow events live. Currently we store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases +- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker +- **Analytics** are performed on the Snowplow events or on the aggregate tables. + + +>  + +## Snowplow schema + +We currently have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/issues/207930) including the following definitions: + +- Frontend and backend taxonomy as listed below +- [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy) +- [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) +- [Iglu schema](https://gitlab.com/gitlab-org/iglu/) +- [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) + +## Enabling Snowplow + +Tracking can be enabled at: + +- The instance level, which will enable tracking on both the frontend and backend layers. +- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser will also not be tracked from a user level. + +We utilize Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: + +- **Admin Area > Settings > Integrations** in the UI. +- `admin/application_settings/integrations` in your browser. + +The following configuration is required: + +| Name | Value | +| ------------- | ------------------------- | +| Collector | `snowplow.trx.gitlab.net` | +| Site ID | `gitlab` | +| Cookie domain | `.gitlab.com` | + +## Snowplow request flow + +The following example shows a basic request/response flow between a Snowplow JS / Ruby Trackers on GitLab.com, [the GitLab.com Snowplow Collector](https://about.gitlab.com/handbook/engineering/infrastructure/library/snowplow/), GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense.: + +```mermaid +sequenceDiagram + participant Snowplow JS (Frontend) + participant Snowplow Ruby (Backend) + participant GitLab.com Snowplow Collector + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + Snowplow JS (Frontend) ->> GitLab.com Snowplow Collector: FE Tracking event + Snowplow Ruby (Backend) ->> GitLab.com Snowplow Collector: BE Tracking event + loop Process using Kinesis Stream + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Log raw events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Enrich events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Write to disk + end + GitLab.com Snowplow Collector ->> S3 Bucket: Kinesis Firehose + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying +``` + +## Implementing Snowplow JS (Frontend) tracking + +GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). + +| field | type | default value | description | +|:-----------|:-------|:---------------------------|:------------| +| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | +| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | + +### Tracking in HAML (or Vue Templates) + +When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute will automatically have event tracking bound on clicks. + +Below is an example of `data-track-*` attributes assigned to a button: + +```haml +%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } +``` + +```html +<button class="btn" + data-track-event="click_button" + data-track-label="template_preview" + data-track-property="my-template" +/> +``` + +Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows for them to be properly handled on rerendering and changes to the DOM, but it's important to know that because of the way these events are bound, click events shouldn't be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you'll need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). + +Below is a list of supported `data-track-*` attributes: + +| attribute | required | description | +|:----------------------|:---------|:------------| +| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | +| `data-track-label` | false | The `label` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | +| `data-track-property` | false | The `property` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | +| `data-track-value` | false | The `value` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). If omitted, this will be the element's `value` property or an empty string. For checkboxes, the default value will be the element's checked attribute or `false` when unchecked. | +| `data-track-context` | false | The `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | + +### Tracking within Vue components + +There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. + +```javascript +import Tracking from '~/tracking'; +const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); +``` + +You can provide default options that will be passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. + +You can then use the mixin normally in your component with the `mixin`, Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These will override any defaults and allows the values to be dynamic from props, or based on state. + +```javascript +export default { + mixins: [trackingMixin], + // ...[component implementation]... + data() { + return { + expanded: false, + tracking: { + label: 'left_sidebar' + } + }; + }, +} +``` + +The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. + +```javascript +export default { + mixins: [Tracking.mixin({ label: 'right_sidebar' })], + data() { + return { + expanded: false, + }; + }, + methods: { + toggle() { + this.expanded = !this.expanded; + this.track('click_toggle', { value: this.expanded }) + } + } +}; +``` + +And if needed within the template, you can use the `track` method directly as well. + +```html +<template> + <div> + <a class="toggle" @click.prevent="toggle">Toggle</a> + <div v-if="expanded"> + <p>Hello world!</p> + <a @click.prevent="track('click_action')">Track an event</a> + </div> + </div> +</template> +``` + +### Tracking in raw JavaScript + +Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. + +```javascript +import Tracking from '~/tracking'; + +const button = document.getElementById('create_from_template_button'); +button.addEventListener('click', () => { + Tracking.event('dashboard:projects:index', 'click_button', { + label: 'create_from_template', + property: 'template_preview', + value: 'rails', + }); +}) +``` + +### Tests and test helpers + +In Jest particularly in vue tests, you can use the following: + +```javascript +import { mockTracking } from 'helpers/tracking_helper'; + +describe('MyTracking', () => { + let spy; + + beforeEach(() => { + spy = mockTracking('_category_', wrapper.element, jest.spyOn); + }); + + it('tracks an event when clicked on feedback', () => { + wrapper.find('.discover-feedback-icon').trigger('click'); + + expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { + label: 'security-discover-feedback-cta', + property: '0', + }); + }); +}); + +``` + +In obsolete Karma tests it's used as below: + +```javascript +import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; + +describe('my component', () => { + let trackingSpy; + + beforeEach(() => { + trackingSpy = mockTracking('_category_', vm.$el, spyOn); + }); + + const triggerEvent = () => { + // action which should trigger a event + }; + + it('tracks an event when toggled', () => { + expect(trackingSpy).not.toHaveBeenCalled(); + + triggerEvent('a.toggle'); + + expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { + label: 'right_sidebar', + property: 'confidentiality', + }); + }); +}); +``` + +## Implementing Snowplow Ruby (Backend) tracking + +GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. + +Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: + +| argument | type | default value | description | +|:-----------|:-------|:---------------------------|:------------| +| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | +| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). These will be set as empty strings if you don't provide them. | + +Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visual performance over time in an area or aspect of code. + +For example: + +```ruby +class Projects::CreateService < BaseService + def execute + project = Project.create(params) + + Gitlab::Tracking.event('Projects::CreateService', 'create_project', + label: project.errors.full_messages.to_sentence, + value: project.valid? + ) + end +end +``` + +### Performance + +We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. + +## Developing and testing Snowplow + +There are several tools for developing and testing Snowplow Event + +| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | +| ------ | ------ | ------ | ------ | ------ | +| Snowplow Analytics Debugger Chrome Extension | ✅ | ❌ | ✅ | ✅ | +| Snowplow Inspector Chrome Extension | ✅ | ❌ | ✅ | ✅ | +| Snowplow Micro | ✅ | ✅ | ✅ | ❌ | +| Snowplow Mini | ✅ | ✅ | ❌ | ✅ | + +### Snowplow Analytics Debugger Chrome Extension + +Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) chrome browser extension +1. Open Chrome DevTools to the Snowplow Analytics Debugger tab +1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html) + +### Snowplow Inspector Chrome Extension + +Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en) +1. Open the chrome extension by pressing the Snowplow Inspector icon beside the address bar +1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. + +### Snowplow Micro + +Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. + +Snowplow Micro is a docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. + +- Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) +- Look at the [Snowplow Micro repo](https://github.com/snowplow-incubator/snowplow-micro) +- Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) + +1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro) + +``` bash +docker run --mount type=bind,source=$(pwd)/example,destination=/config -p 9090:9090 snowplow/snowplow-micro:latest --collector-config /config/micro.conf --iglu /config/iglu.json +``` + +1. Install snowplow micro by cloning the settings in [this project](https://gitlab.com/a_akgun/snowplow-micro). + + ``` bash + git clone git@gitlab.com:a_akgun/snowplow-micro.git + ./snowplow-micro.sh + ``` + +1. Update port in SQL (needed to set 9090) + + ``` bash + gdk psql -d gitlabhq_development + update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; + ``` + +1. Update `app/assets/javascripts/tracking.js` to [remove this line](https://gitlab.com/snippets/1918635): + + ``` javascript + forceSecureTracker: true + ``` + +1. Update `lib/gitlab/tracking.rb` to [add these lines](https://gitlab.com/snippets/1918635): + + ``` ruby + protocol: 'http', + port: 9090, + ``` + +1. Update `lib/gitlab/tracking.rb` to [change async emitter from https to http](https://gitlab.com/snippets/1918635): + + ``` ruby + SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'http'), + ``` + +1. Enable Snowplow in the admin area, Settings::Integrations::Snowplow to point to: + `http://localhost:3000/admin/application_settings/integrations#js-snowplow-settings` +1. `gdk restart` +1. Send a test Snowplow event from the Rails console + + ``` ruby + Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', { page_type: ‘MY_TYPE' }, context: nil ) + ``` + +### Snowplow Mini + +[Snowplow Mini](https://github.com/snowplow/snowplow-mini) is an easily-deployable, single-instance version of Snowplow. + +Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. + +For GitLab.com, we are currently setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. diff --git a/doc/development/telemetry/usage_ping.md b/doc/development/telemetry/usage_ping.md new file mode 100644 index 00000000000..e9b959eaa96 --- /dev/null +++ b/doc/development/telemetry/usage_ping.md @@ -0,0 +1,489 @@ +# Usage Ping Guide + +> - [Introduced][ee-557] in GitLab Enterprise Edition 8.10. +> - More statistics [were added][ee-735] in GitLab Enterprise Edition 8.12. +> - [Moved to GitLab Core][ce-23361] in 9.1. +> - More statistics [were added][ee-6602] in GitLab Ultimate 11.2. + +This guide provides a details about how usage ping works. It includes the following sections: + +1. [What is Usage Ping](#what-is-usage-ping) +1. [Usage Ping payload](#usage-ping-payload) +1. [Disabling Usage Ping](#disabling-usage-ping) +1. [Usage Ping request flow](#usage-ping-request-flow) +1. [How Usage Ping works](#how-usage-ping-works) +1. [Implementing Usage Ping](#implementing-usage-ping) +1. [Developing and testing usage ping](#developing-and-testing-usage-ping) + +For more information about Telemetry, see: + +- [Telemetry Guide](index.md) +- [Snowplow Guide](snowplow.md) + +More useful links: + +- [Telemetry Direction](https://about.gitlab.com/direction/telemetry/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#-data-analysis-process) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/) + +## What is Usage Ping + +- GitLab sends a weekly payload containing usage data to GitLab Inc. The usage ping uses high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. +- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. +- Usage ping is important to GitLab as we use it to calculate our and Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. +- Once usage ping is enabled, GitLab will gather data from the other instances and will be able to show usage statistics of your instance to your users. + +### Why Should We Enable Usage Ping? + +- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we are able to make better product decisions. +- As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. +- As a benefit of having the usage ping active, GitLab provides you with The DevOps Score,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. +- You will get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) +- You will get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? +- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. + +### Limitations + +- Usage Ping does not track frontend events things like page views, link clicks, or user sessions and only focuses on aggregated backend events. +- Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. + +## Usage Ping payload + +You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: + +1. Navigate to the **Admin Area > Settings > Metrics and profiling**. +1. Expand the **Usage statistics** section. +1. Click the **Preview payload** button. + +Here is an example of the payload structure + +``` json +{ + "uuid": "0000000-0000-0000-0000-000000000000", + "hostname": "example.com", + "version": "12.10.0-pre", + "installation_type": "omnibus-gitlab", + "active_user_count": 999, + "recorded_at": "2020-04-17T07:43:54.162+00:00", + "edition": "EEU", + "license_md5": "00000000000000000000000000000000", + "license_id": null, + "historical_max_users": 999, + "licensee": { + "Name": "ABC, Inc.", + "Email": "email@example.com", + "Company": "ABC, Inc." + }, + "license_user_count": 999, + "license_starts_at": "2020-01-01", + "license_expires_at": "2021-01-01", + "license_plan": "ultimate", + "license_add_ons": { + }, + "license_trial": false, + "counts": { + "assignee_lists": 999, + "boards": 999, + "ci_builds": 999, + ... + }, + "container_registry_enabled": true, + "dependency_proxy_enabled": false, + "gitlab_shared_runners_enabled": true, + "gravatar_enabled": true, + "influxdb_metrics_enabled": true, + "ldap_enabled": false, + "mattermost_enabled": false, + "omniauth_enabled": true, + "prometheus_metrics_enabled": false, + "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", + "signup_enabled": true, + "web_ide_clientside_preview_enabled": true, + "ingress_modsecurity_enabled": true, + "projects_with_expiration_policy_disabled": 999, + "projects_with_expiration_policy_enabled": 999, + ... + "elasticsearch_enabled": true, + "license_trial_ends_on": null, + "geo_enabled": false, + "git": { + "version": { + "major": 2, + "minor": 26, + "patch": 1 + } + }, + "gitaly": { + "version": "12.10.0-rc1-93-g40980d40", + "servers": 56, + "filesystems": [ + "EXT_2_3_4" + ] + }, + "gitlab_pages": { + "enabled": true, + "version": "1.17.0" + }, + "database": { + "adapter": "postgresql", + "version": "9.6.15" + }, + "app_server": { + "type": "console" + }, + "avg_cycle_analytics": { + "issue": { + "average": 999, + "sd": 999, + "missing": 999 + }, + "plan": { + "average": null, + "sd": 999, + "missing": 999 + }, + "code": { + "average": null, + "sd": 999, + "missing": 999 + }, + "test": { + "average": null, + "sd": 999, + "missing": 999 + }, + "review": { + "average": null, + "sd": 999, + "missing": 999 + }, + "staging": { + "average": null, + "sd": 999, + "missing": 999 + }, + "production": { + "average": null, + "sd": 999, + "missing": 999 + }, + "total": 999 + }, + "usage_activity_by_stage": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "usage_activity_by_stage_monthly": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + } +} +``` + +## Disabling usage ping + +The usage ping is opt-out. If you want to deactivate this feature, go to the Settings page of your administration panel and uncheck the Usage Ping checkbox. + +To disable the usage ping and prevent it from being configured in future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): + +```ruby +gitlab_rails['usage_ping_enabled'] = false +``` + +And source installs can set the following in `gitlab.yml`: + +```yaml +production: &base + # ... + gitlab: + # ... + usage_ping_enabled: false +``` + +## Usage Ping Request Flow + +The following example shows a basic request/response flow between a GitLab Instance, the Versions Application, the License Application, Salesforce, GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense.: + +```mermaid +sequenceDiagram + participant GitLab Instance + participant Versions Application + participant Licenses Application + participant Salesforce + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + GitLab Instance->>Versions Application: Send Usage Ping + loop Process usage data + Versions Application->>Versions Application: Parse usage data + Versions Application->>Versions Application: Write to database + Versions Application->>Versions Application: Update license ping time + end + loop Process data for Salesforce + Versions Application-xLicenses Application: Request Zuora subscription id + Licenses Application-xVersions Application: Zuora subscription id + Versions Application-xSalesforce: Request Zuora account id by Zuora subscription id + Salesforce-xVersions Application: Zuora account id + Versions Application-xSalesforce: Usage data for the Zuora account + end + Versions Application->>S3 Bucket: Export Versions database + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying + Versions Application->>GitLab Instance: DevOps Score (Conversational Development Index) +``` + +## How Usage Ping works + +1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. +1. When the cron job runs, it calls [GitLab::UsageData.to_json](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). +1. GitLab::UsageData.to_json [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. +1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in GitLab::UsageData.to_json. +1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20). + +## Implementing Usage Ping + +Usage Ping consists of four types of counters which are all found in `usage_data.rb`: + +- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation +- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column +- **Alternative Counters:** Used for settings and configurations +- **Redis Counters:** Used for in-memory counts. This method is being deprecated due to data inaccuracies and will be replaced with a persistent method. + +Note: Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. + +### Why batch counting + +For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. + +For GitLab.com, there are extremely large tables with 15 second query timeouts, so, we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: + +| Table | Row counts in millions | +| ------ | ------ | +| merge_request_diff_commits | 2280 | +| ci_build_trace_sections | 1764 | +| merge_request_diff_files | 1082 | +| events | 514 | + +There are two batch counting methods provided, `Ordinary Batch Counters` and `Distinct Batch Counters`. Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, a specialized index may need to be added on the columns involved in a counter. + +### Ordinary Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Simple count of a given ActiveRecord_Relation + +Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the count on, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +count(User.active) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id)) +``` + +### Distinct Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Distinct count of a given ActiveRecord_Relation on given column + +Method: `distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the distinct count, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +distinct_count(::Project, :creator_id) +distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') +``` + +### Redis Counters + +Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent` +returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent +different behavior due to 2 different implementations of Redis counter + +Method: `redis_usage_data(counter, &block)` + +Arguments: + +- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented +- or a `block`: wich is evaluated + +Example of usage: + +```ruby +redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) +redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } +``` + +Note that Redis counters are in the [process of being deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/216330) and you should instead try to use Snowplow events instead. We're in the process of building [self-managed event tracking](https://gitlab.com/gitlab-org/telemetry/-/issues/373) and once this is available, we will convert all Redis counters into Snowplow events. + +### Alternative Counters + +Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception. +Mainly used for settings and configurations. + +Method: `alt_usage_data(value = nil, fallback: -1, &block)` + +Arguments: + +- `value`: a simple static value in wich case the value is simply returned. +- or a `block`: wich is evaluated +- `fallback: -1`: the common value used for any metrics that are failing. + +Example of usage: + +```ruby +alt_usage_data { Gitlab::VERSION } +alt_usage_data { Gitlab::CurrentSettings.uuid } +alt_usage_data(999) +``` + +## Developing and testing Usage Ping + +### 1. Use your Rails console to manually test counters + +```ruby +# count +Gitlab::UsageData.count(User.active) +Gitlab::UsageData.count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) + +# count distinct +Gitlab::UsageData.distinct_count(::Project, :creator_id) +Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +``` + +### 2. Generate the SQL query + +Your Rails console will give back the generated SQL queries. + +Example: + +```ruby + pry(main)> Gitlab::UsageData.count(User.active) + (0.4ms) SELECT "features"."key" FROM "features" + (0.7ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3)) + (0.6ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3)) + (0.5ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3)) AND "users"."id" BETWEEN 0 AND 99999 +``` + +### 3. Optimize queries with #database-lab + +Paste the SQL query into `#database-lab` to see how the query performs at scale. + +- #database-lab is a Slack channel which uses a production-sized environment to test your queries +- GitLab.com’s production database has a 15 second timeout. +- For each query we require an execution time of under 1 second due do cold caches which can 10x this time. +- Add a specialized index on columns involved to reduce your the execution time. + +In order to have an understanding of the queries execution we add in the MR description the following information + +For counters that have a `time_period` test and add information for both cases. + +- with `time_period = {}` for all time period +- and `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period + +Execution plan and query time before and after optimization + +Using database-lab and [explain.depesz.com](https://explain.depesz.com/) see more details in [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries) + +Query generated for the index and time + +Using database-lab + +Migration output for up and down execution + +Examples of query optimization work: + +- [Example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445) +- [Example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871) + +### 4. Ask for a Telemetry Review + +On GitLab.com, we have DangerBot setup to monitor Telemetry related files and DangerBot will recommend a Telemetry review. Simply `@gitlab-org/growth/telemetry/engineers` in your MR for a review. diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 62767180077..f0137e542cc 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -17,24 +17,22 @@ our test design. We can find some helpful heuristics documented in the Handbook ## Test speed -GitLab has a massive test suite that, without [parallelization], can take hours +GitLab has a massive test suite that, without [parallelization](ci.md#test-suite-parallelization-on-the-ci), can take hours to run. It's important that we make an effort to write tests that are accurate and effective _as well as_ fast. Here are some things to keep in mind regarding test performance: -- `double` and `spy` are faster than `FactoryBot.build(...)` +- `instance_double` and `spy` are faster than `FactoryBot.build(...)` - `FactoryBot.build(...)` and `.build_stubbed` are faster than `.create`. - Don't `create` an object when `build`, `build_stubbed`, `attributes_for`, - `spy`, or `double` will do. Database persistence is slow! + `spy`, or `instance_double` will do. Database persistence is slow! - Don't mark a feature as requiring JavaScript (through `:js` in RSpec) unless it's _actually_ required for the test to be valid. Headless browser testing is slow! -[parallelization]: ci.md#test-suite-parallelization-on-the-ci - ## RSpec -To run rspec tests: +To run RSpec tests: ```shell # run all tests @@ -71,6 +69,8 @@ FDOC=1 bin/rspec spec/[path]/[to]/[spec].rb - Use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'` - Don't assert against the absolute value of a sequence-generated attribute (see [Gotchas](../gotchas.md#do-not-assert-against-the-absolute-value-of-a-sequence-generated-attribute)). +- Avoid using `expect_any_instance_of` or `allow_any_instance_of` (see + [Gotchas](../gotchas.md#do-not-assert-against-the-absolute-value-of-a-sequence-generated-attribute)). - Don't supply the `:each` argument to hooks since it's the default. - On `before` and `after` hooks, prefer it scoped to `:context` over `:all` - When using `evaluate_script("$('.js-foo').testSomething()")` (or `execute_script`) which acts on a given element, @@ -320,26 +320,48 @@ stub_feature_flags(ci_live_trace: false) Feature.enabled?(:ci_live_trace) # => false ``` -If you wish to set up a test where a feature flag is disabled for some -actors and not others, you can specify this in options passed to the -helper. For example, to disable the `ci_live_trace` feature flag for a -specifc project: +If you wish to set up a test where a feature flag is enabled only +for some actors and not others, you can specify this in options +passed to the helper. For example, to enable the `ci_live_trace` +feature flag for a specifc project: ```ruby project1, project2 = build_list(:project, 2) -# Feature will only be disabled for project1 -stub_feature_flags(ci_live_trace: { enabled: false, thing: project1 }) +# Feature will only be enabled for project1 +stub_feature_flags(ci_live_trace: project1) + +Feature.enabled?(:ci_live_trace) # => false +Feature.enabled?(:ci_live_trace, project1) # => true +Feature.enabled?(:ci_live_trace, project2) # => false +``` + +This represents an actual behavior of FlipperGate: -Feature.enabled?(:ci_live_trace, project1) # => false -Feature.enabled?(:ci_live_trace, project2) # => true +1. You can enable an override for a specified actor to be enabled +1. You can disable (remove) an override for a specified actor, + fallbacking to default state +1. There's no way to model that you explicitly disable a specified actor + +```ruby +Feature.enable(:my_feature) +Feature.disable(:my_feature, project1) +Feature.enabled?(:my_feature) # => true +Feature.enabled?(:my_feature, project1) # => true +``` + +```ruby +Feature.disable(:my_feature2) +Feature.enable(:my_feature2, project1) +Feature.enabled?(:my_feature2) # => false +Feature.enabled?(:my_feature2, project1) # => true ``` ### Pristine test environments The code exercised by a single GitLab test may access and modify many items of data. Without careful preparation before a test runs, and cleanup afterward, -data can be changed by a test in such a way that it affects the behaviour of +data can be changed by a test in such a way that it affects the behavior of following tests. This should be avoided at all costs! Fortunately, the existing test framework handles most cases already. @@ -493,7 +515,7 @@ range of inputs. By specifying the test case once, alongside a table of inputs and the expected output for each, your tests can be made easier to read and more compact. -We use the [rspec-parameterized](https://github.com/tomykaira/rspec-parameterized) +We use the [RSpec::Parameterized](https://github.com/tomykaira/rspec-parameterized) gem. A short example, using the table syntax and checking Ruby equality for a range of inputs, might look like this: @@ -526,7 +548,7 @@ objects, FactoryBot-created objects etc. can lead to ### Prometheus tests Prometheus metrics may be preserved from one test run to another. To ensure that metrics are -reset before each example, add the `:prometheus` tag to the Rspec test. +reset before each example, add the `:prometheus` tag to the RSpec test. ### Matchers @@ -651,7 +673,7 @@ end ### Factories -GitLab uses [factory_bot] as a test fixture replacement. +GitLab uses [factory_bot](https://github.com/thoughtbot/factory_bot) as a test fixture replacement. - Factory definitions live in `spec/factories/`, named using the pluralization of their corresponding model (`User` factories are defined in `users.rb`). @@ -666,8 +688,6 @@ GitLab uses [factory_bot] as a test fixture replacement. - Factories don't have to be limited to `ActiveRecord` objects. [See example](https://gitlab.com/gitlab-org/gitlab-foss/commit/0b8cefd3b2385a21cfed779bd659978c0402766d). -[factory_bot]: https://github.com/thoughtbot/factory_bot - ### Fixtures All fixtures should be placed under `spec/fixtures/`. diff --git a/doc/development/testing_guide/end_to_end/beginners_guide.md b/doc/development/testing_guide/end_to_end/beginners_guide.md new file mode 100644 index 00000000000..73960a2f74d --- /dev/null +++ b/doc/development/testing_guide/end_to_end/beginners_guide.md @@ -0,0 +1,340 @@ +# Beginner's guide to writing end-to-end tests + +In this tutorial, you will learn about the creation of end-to-end (_e2e_) tests +for [GitLab Community Edition](https://about.gitlab.com/install/?version=ce) and +[GitLab Enterprise Edition](https://about.gitlab.com/install/). + +By the end of this tutorial, you will be able to: + +- Determine whether an end-to-end test is needed. +- Understand the directory structure within `qa/`. +- Write a basic end-to-end test that will validate login features. +- Develop any missing [page object](page_objects.md) libraries. + +## Before you write a test + +Before you write tests, your +[GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit) +must be configured to run the specs. The end-to-end tests: + +- Are contained within the `qa/` directory. +- Should be independent and + [idempotent](https://en.wikipedia.org/wiki/Idempotence#Computer_science_meaning). +- Create [resources](resources.md) (such as project, issue, user) on an ad-hoc basis. +- Test the UI and API interfaces, and use the API to efficiently set up the UI tests. + +TIP: **Tip:** +For more information, see [End-to-end testing Best Practices](best_practices.md). + +## Determine if end-to-end tests are needed + +Check the code coverage of a specific feature before writing end-to-end tests, +for both [GitLab Community Edition](https://gitlab-org.gitlab.io/gitlab-foss/coverage-ruby/#_AllFiles) +and [GitLab Enterprise Edition](https://gitlab-org.gitlab.io/gitlab/coverage-ruby/#_AllFiles) projects. +Does sufficient test coverage exist at the unit, feature, or integration levels? +If you answered *yes*, then you *don't* need an end-to-end test. + +For information about the distribution of tests per level in GitLab, see +[Testing Levels](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/testing_guide/testing_levels.md). + +- See the + [How to test at the correct level?](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/testing_guide/testing_levels.md#how-to-test-at-the-correct-level) + section of the [Testing levels](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/testing_guide/testing_levels.md) document. +- Review how often the feature changes. Stable features that don't change very often + might not be worth covering with end-to-end tests if they are already covered + in lower level tests. +- Finally, discuss the proposed test with the developer(s) involved in implementing + the feature and the lower-level tests. + +CAUTION: **Caution:** +Check both [GitLab Community Edition](https://gitlab-org.gitlab.io/gitlab-foss/coverage-ruby/#_AllFiles) and +[GitLab Enterprise Edition](https://gitlab-org.gitlab.io/gitlab/coverage-ruby/#_AllFiles) coverage projects +for previously-written tests for this feature. For analyzing the code coverage, +you must understand which application files implement specific features. + +NOTE: **Note:** +In this tutorial we're writing a login end-to-end test, even though it has been +sufficiently covered by lower-level testing, because it's the first step for most +end-to-end flows, and is easiest to understand. + +## Identify the DevOps stage + +The GitLab QA end-to-end tests are organized by the different +[stages in the DevOps lifecycle](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features/browser_ui). +Determine where the test should be placed by +[stage](https://about.gitlab.com/handbook/product/categories/#devops-stages), +determine which feature the test will belong to, and then place it in a subdirectory +under the stage. + + + +NOTE: **Note:** +If the test is Enterprise Edition only, the test will be created in the `features/ee` +directory, but follow the same DevOps lifecycle format. + +## Create a skeleton test + +In the first part of this tutorial we will be testing login, which is owned by the +Manage stage. Inside `qa/specs/features/browser_ui/1_manage/login`, create a +file `basic_login_spec.rb`. + +### The outer `context` block + +Specs have an outer `context` indicating the DevOps stage. + +```ruby +# frozen_string_literal: true + +module QA + context 'Manage' do + + end +end +``` + +### The `describe` block + +Inside of our outer `context`, describe the feature to test. In this case, `Login`. + +```ruby +# frozen_string_literal: true + +module QA + context 'Manage' do + describe 'Login' do + + end + end +end +``` + +### The `it` blocks (examples) + +Every test suite contains at least one `it` block (example). A good way to start +writing end-to-end tests is to write test case descriptions as `it` blocks: + +```ruby +module QA + context 'Manage' do + describe 'Login' do + it 'can login' do + + end + + it 'can logout' do + + end + end + end +end +``` + +## Write the test + +An important question is "What do we test?" and even more importantly, "How do we test?" + +Begin by logging in. + +```ruby +# frozen_string_literal: true + +module QA + context 'Manage' do + describe 'Login' do + it 'can login' do + Flow::Login.sign_in + + end + + it 'can logout' do + Flow::Login.sign_in + + end + end + end +end +``` + +After [running the spec](#run-the-spec), our test should login and end; then we +should answer the question "What do we test?" + +```ruby +# frozen_string_literal: true + +module QA + context 'Manage' do + describe 'Login' do + it 'can login' do + Flow::Login.sign_in + + Page::Main::Menu.perform do |menu| + expect(menu).to be_signed_in + end + end + + it 'can logout' do + Flow::Login.sign_in + + Page::Main::Menu.perform do |menu| + menu.sign_out + + expect(menu).not_to be_signed_in + end + end + end + end +end +``` + +**What do we test?** + +1. Can we log in? +1. Can we log out? + +**How do we test?** + +1. Check if the user avatar appears in the top navigation. +1. Check if the user avatar *does not* appear in the top navigation. + +NOTE: **Note:** +Behind the scenes, `be_signed_in` is a +[predicate matcher](https://relishapp.com/rspec/rspec-expectations/v/3-8/docs/built-in-matchers/predicate-matchers) +that [implements checking the user avatar](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/page/main/menu.rb#L74). + +## De-duplicate your code + +Refactor your test to use a `before` block for test setup, since it's duplicating +a call to `sign_in`. + +```ruby +# frozen_string_literal: true + +module QA + context 'Manage' do + describe 'Login' do + before do + Flow::Login.sign_in + end + + it 'can login' do + Page::Main::Menu.perform do |menu| + expect(menu).to be_signed_in + end + end + + it 'can logout' do + Page::Main::Menu.perform do |menu| + menu.sign_out + + expect(menu).not_to be_signed_in + end + end + end + end +end +``` + +The `before` block is essentially a `before(:each)` and is run before each example, +ensuring we now log in at the beginning of each test. + +## Test setup using resources and page objects + +Next, let's test something other than Login. Let's test Issues, which are owned by the Plan +stage, so [create a file](#identify-the-devops-stage) in +`qa/specs/features/browser_ui/3_create/issues` called `issues_spec.rb`. + +```ruby +# frozen_string_literal: true + +module QA + context 'Plan' do + describe 'Issues' do + let(:issue) do + Resource::Issue.fabricate_via_api! do |issue| + issue.title = 'My issue' + issue.description = 'This is an issue specific to this test' + end + end + + before do + Flow::Login.sign_in + issue.visit! + end + + it 'can close an issue' do + Page::Project::Issue::Show.perform do |show| + show.click_close_issue_button + + expect(show).to be_closed + end + end + end + end +end +``` + +Note the following important points: + +- At the start of our example, we will be at the `page/issue/show.rb` [page](page_objects.md). +- Our test fabricates only what it needs, when it needs it. +- The issue is fabricated through the API to save time. +- GitLab prefers `let()` over instance variables. See + [best practices](../best_practices.md#let-variables). +- `be_closed` is not implemented in `page/project/issue/show.rb` yet, but will be + implemented in the next step. + +The issue is fabricated as a [Resource](resources.md), which is a GitLab entity +you can create through the UI or API. Other examples include: + +- A [Merge Request](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/resource/merge_request.rb). +- A [User](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/resource/user.rb). +- A [Project](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/resource/project.rb). +- A [Group](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/resource/group.rb). + +## Write the page object + +A [Page Object](page_objects.md) is a class in our suite that represents a page +within GitLab. The **Login** page would be one example. Since our page object for +the **Issue Show** page already exists, add the `closed?` method. + +```ruby +module Page::Project::Issue + class Show + view 'app/views/projects/issues/show.html.haml' do + element :closed_status_box + end + + def closed? + has_element?(:closed_status_box) + end + end +end +``` + +Next, define the element `closed_status_box` within your view, so your Page Object +can see it. + +```haml +-#=> app/views/projects/issues/show.html.haml +.issuable-status-box.status-box.status-box-issue-closed{ ..., data: { qa_selector: 'closed_status_box' } } +``` + +## Run the spec + +Before running the spec, confirm: + +- The GDK is installed. +- The GDK is running on port 3000 locally. +- No additional [RSpec metadata tags](rspec_metadata_tests.md) have been applied. +- Your working directory is `qa/` within your GDK GitLab installation. + +To run the spec, run the following command: + +```ruby +bundle exec bin/qa Test::Instance::All http://localhost:3000 -- <test_file> +``` + +Where `<test_file>` is: + +- `qa/specs/features/browser_ui/1_manage/login/login_spec.rb` when running the Login example. +- `qa/specs/features/browser_ui/2_plan/issues/issue_spec.rb` when running the Issue example. diff --git a/doc/development/testing_guide/end_to_end/img/gl-devops-lifecycle-by-stage-numbers_V12_10.png b/doc/development/testing_guide/end_to_end/img/gl-devops-lifecycle-by-stage-numbers_V12_10.png Binary files differnew file mode 100644 index 00000000000..d11305c3686 --- /dev/null +++ b/doc/development/testing_guide/end_to_end/img/gl-devops-lifecycle-by-stage-numbers_V12_10.png diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index 443b7b06a24..e6a683e9148 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -180,7 +180,7 @@ instance-level scenarios](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/ Continued reading: -- [Quick Start Guide](quick_start_guide.md) +- [Beginner's Guide](beginners_guide.md) - [Style Guide](style_guide.md) - [Best Practices](best_practices.md) - [Testing with feature flags](feature_flags.md) diff --git a/doc/development/testing_guide/end_to_end/page_objects.md b/doc/development/testing_guide/end_to_end/page_objects.md index 22e7375be1f..9d4fa5316c4 100644 --- a/doc/development/testing_guide/end_to_end/page_objects.md +++ b/doc/development/testing_guide/end_to_end/page_objects.md @@ -238,6 +238,53 @@ the view for code in a library. In such rare cases it's reasonable to use CSS selectors in page object methods, with a comment explaining why an `element` can't be added. +### Define Page concerns + +Some pages share common behaviors, and/or are prepended with EE-specific modules that adds EE-specific methods. + +These modules must: + +1. Extend from the `QA::Page::PageConcern` module, with `extend QA::Page::PageConcern`. +1. Override the `self.prepended` method if they need to `include`/`prepend` other modules themselves, and/or define + `view` or `elements`. +1. Call `super` as the first thing in `self.prepended`. +1. Include/prepend other modules and define their `view`/`elements` in a `base.class_eval` block to ensure they're + defined in the class that prepends the module. + +These steps ensure the sanity selectors check will detect problems properly. + +For example, `qa/qa/ee/page/merge_request/show.rb` adds EE-specific methods to `qa/qa/page/merge_request/show.rb` (with +`QA::Page::MergeRequest::Show.prepend_if_ee('QA::EE::Page::MergeRequest::Show')`) and following is how it's implemented +(only showing the relevant part and refering to the 4 steps described above with inline comments): + +```ruby +module QA + module EE + module Page + module MergeRequest + module Show + extend QA::Page::PageConcern # 1. + + def self.prepended(base) # 2. + super # 3. + + base.class_eval do # 4. + prepend Page::Component::LicenseManagement + + view 'app/assets/javascripts/vue_merge_request_widget/components/states/sha_mismatch.vue' do + element :head_mismatch, "The source branch HEAD has recently changed." + end + + [...] + end + end + end + end + end + end +end +``` + ## Running the test locally During development, you can run the `qa:selectors` test by running diff --git a/doc/development/testing_guide/end_to_end/quick_start_guide.md b/doc/development/testing_guide/end_to_end/quick_start_guide.md deleted file mode 100644 index 0ae3f375284..00000000000 --- a/doc/development/testing_guide/end_to_end/quick_start_guide.md +++ /dev/null @@ -1,621 +0,0 @@ -# Writing end-to-end tests step-by-step - -In this tutorial, you will find different examples, and the steps involved, in the creation of end-to-end (_e2e_) tests for GitLab CE and GitLab EE, using GitLab QA. - -When referring to end-to-end tests in this document, this means testing a specific feature end-to-end such as: - -- A user logging in. -- The creation of a project. -- The management of labels. -- Breaking down epics into sub-epics and issues. - -## Important information before we start writing tests - -It's important to understand that end-to-end tests of isolated features, such as the ones described in the above note, doesn't mean that everything needs to happen through the GUI. - -If you don't exactly understand what we mean by **not everything needs to happen through the GUI,** please make sure you've read the [best practices](best_practices.md) before moving on. - -## This document covers the following items - -- [0.](#0-are-end-to-end-tests-needed) Identifying if end-to-end tests are really needed -- [1.](#1-identifying-the-devops-stage) Identifying the [DevOps stage](https://about.gitlab.com/stages-devops-lifecycle/) of the feature that you are going to cover with end-to-end tests -- [2.](#2-test-skeleton) Creating the skeleton of the test file (`*_spec.rb`) -- [3.](#3-test-cases-mvc) The [MVC](https://about.gitlab.com/handbook/values/#minimum-viable-change-mvc) of the test cases' logic -- [4.](#4-extracting-duplicated-code) Extracting duplicated code into methods -- [5.](#5-tests-pre-conditions-using-resources-and-page-objects) Tests' pre-conditions (`before :context` and `before`) using resources and [Page Objects](page_objects.md) -- [6.](#6-optimization) Optimizing the test suite -- [7.](#7-resources) Using and implementing resources -- [8.](#8-page-objects) Moving element definitions and methods to [Page Objects](page_objects.md) - -### 0. Are end-to-end tests needed? - -At GitLab we respect the [test pyramid](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/testing_guide/testing_levels.md), and so, we recommend you check the code coverage of a specific feature before writing end-to-end tests, for both [CE](https://gitlab-org.gitlab.io/gitlab-foss/coverage-ruby/#_AllFiles) and [EE](https://gitlab-org.gitlab.io/gitlab/coverage-ruby/#_AllFiles) projects. - -Sometimes you may notice that there is already good coverage in lower test levels, and we can stay confident that if we break a feature, we will still have quick feedback about it, even without having end-to-end tests. - -> For analyzing the code coverage, you will also need to understand which application files implement specific functionalities. - -#### Some other guidelines are as follows - -- Take a look at the [How to test at the correct level?](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/testing_guide/testing_levels.md#how-to-test-at-the-correct-level) section of the [Testing levels](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/testing_guide/testing_levels.md) document - -- Look into the frequency in which such a feature is changed (_Stable features that don't change very often might not be worth covering with end-to-end tests if they're already covered in lower levels_) - -- Finally, discuss with the developer(s) involved in developing the feature and the tests themselves, to get their feeling - -If after this analysis you still think that end-to-end tests are needed, keep reading. - -### 1. Identifying the DevOps stage - -The GitLab QA end-to-end tests are organized by the different [stages in the DevOps lifecycle](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features/browser_ui), and so, if you are creating tests for issue creation, for instance, you would locate the spec files under the `qa/qa/specs/features/browser_ui/2_plan/` directory since issue creation is part of the Plan stage. - - In another case of a test for listing merged merge requests (MRs), the test should go under the `qa/qa/specs/features/browser_ui/3_create/` directory since merge requests are a feature from the Create stage. - -> There may be sub-directories inside the stages directories, for different features. For example: `.../browser_ui/2_plan/ee_epics/` and `.../browser_ui/2_plan/issues/`. - -Now, let's say we want to create tests for the [scoped labels](https://about.gitlab.com/releases/2019/04/22/gitlab-11-10-released/#scoped-labels) feature, available on GitLab EE Premium (this feature is part of the Plan stage.) - -> Because these tests are for a feature available only on GitLab EE, we need to create them in the [EE repository](https://gitlab.com/gitlab-org/gitlab). - -Since [there is no specific directory for this feature](https://gitlab.com/gitlab-org/gitlab/tree/master/qa/qa/specs/features/browser_ui/2_plan), we should create a sub-directory for it. - -Under `.../browser_ui/2_plan/`, let's create a sub-directory called `ee_scoped_labels/`. - -> Notice that since this feature is only available for GitLab EE we prefix the sub-directory with `ee_`. - -### 2. Test skeleton - -Inside the newly created sub-directory, let's create a file describing the test suite (e.g. `editing_scoped_labels_spec.rb`.) - -#### The `context` and `describe` blocks - -Specs have an outer `context` that indicates the DevOps stage. The next level is the `describe` block, that briefly states the subject of the test suite. See the following example: - -```ruby -module QA - context 'Plan' do - describe 'Editing scoped labels on issues' do - end - end -end -``` - -#### The `it` blocks - -Every test suite is composed of at least one `it` block, and a good way to start writing end-to-end tests is by writing test cases descriptions as `it` blocks. These might help you to think of different test scenarios. Take a look at the following example: - -```ruby -module QA - context 'Plan' do - describe 'Editing scoped labels on issues' do - it 'replaces an existing label if it has the same key' do - end - - it 'keeps both scoped labels when adding a label with a different key' do - end - end - end -end -``` - -### 3. Test cases MVC - -For the [MVC](https://about.gitlab.com/handbook/values/#minimum-viable-change-mvc) of our test cases, let's say that we already have the application in the state needed for the tests, and then let's focus on the logic of the test cases only. - -To evolve the test cases drafted on step 2, let's imagine that the user is already logged into a GitLab EE instance, they already have at least a Premium license in use, there is already a project created, there is already an issue opened in the project, the issue already has a scoped label (e.g. `animal::fox`), there are other scoped labels (for the same scope and for a different scope (e.g. `animal::dolphin` and `plant::orchid`), and finally, the user is already on the issue's page. Let's also suppose that for every test case the application is in a clean state, meaning that one test case won't affect another. - -> Note: there are different approaches to creating an application state for end-to-end tests. Some of them are very time consuming and subject to failures, such as when using the GUI for all the pre-conditions of the tests. On the other hand, other approaches are more efficient, such as using the public APIs. The latter is more efficient since it doesn't depend on the GUI. We won't focus on this part yet, but it's good to keep it in mind. - -Let's now focus on the first test case. - -```ruby -it 'replaces an existing label if it has the same key' do - # This implementation is only for tutorial purposes. We normally encapsulate elements in Page Objects (which we cover on section 8). - page.find('.block.labels .edit-link').click - page.find('.dropdown-menu-labels .dropdown-input-field').send_keys ['animal::dolphin', :enter] - page.find('#content-body').click - page.refresh - - labels_block = page.find(%q([data-qa-selector="labels_block"])) - - expect(labels_block).to have_content('animal::dolphin') - expect(labels_block).not_to have_content('animal::fox') - expect(page).to have_content('added animal::dolphin label and removed animal::fox') -end -``` - -> Notice that the test itself is simple. The most challenging part is the creation of the application state, which will be covered later. -> -> The exemplified test case's MVC is not enough for the change to be merged, but it helps to build up the test logic. The reason is that we do not want to use locators directly in the tests, and tests **must** use [Page Objects](page_objects.md) before they can be merged. This way we better separate the responsibilities, where the Page Objects encapsulate elements and methods that allow us to interact with pages, while the spec files describe the test cases in more business-related language. - -Below are the steps that the test covers: - -1. The test finds the 'Edit' link for the labels and clicks on it. -1. Then it fills in the 'Assign labels' input field with the value 'animal::dolphin' and press enters. -1. Then it clicks in the content body to apply the label and refreshes the page. -1. Finally, the expectations check that the previous scoped label was removed and that the new one was added. - -Let's now see how the second test case would look. - -```ruby -it 'keeps both scoped labels when adding a label with a different key' do - # This implementation is only for tutorial purposes. We normally encapsulate elements in Page Objects (which we cover on section 8). - page.find('.block.labels .edit-link').click - page.find('.dropdown-menu-labels .dropdown-input-field').send_keys ['plant::orchid', :enter] - page.find('#content-body').click - page.refresh - - labels_block = page.find(%q([data-qa-selector="labels_block"])) - - expect(labels_block).to have_content('animal::fox') - expect(labels_block).to have_content('plant::orchid') - expect(page).to have_content('added animal::fox') - expect(page).to have_content('added plant::orchid') -end -``` - -> Note that elements are always located using CSS selectors, and a good practice is to add test-specific selectors (this is called "testability"). For example, the `labels_block` element uses the CSS selector [`data-qa-selector="labels_block"`](page_objects.md#data-qa-selector-vs-qa-selector), which was added specifically for testing purposes. - -Below are the steps that the test covers: - -1. The test finds the 'Edit' link for the labels and clicks on it. -1. Then it fills in the 'Assign labels' input field with the value 'plant::orchid' and press enters. -1. Then it clicks in the content body to apply the label and refreshes the page. -1. Finally, the expectations check that both scoped labels are present. - -> Similar to the previous test, this one is also very straightforward, but there is some code duplication. Let's address it. - -### 4. Extracting duplicated code - -If we refactor the tests created on step 3 we could come up with something like this: - -```ruby -before do - ... - - @initial_label = 'animal::fox' - @new_label_same_scope = 'animal::dolphin' - @new_label_different_scope = 'plant::orchid' - - ... -end - -it 'replaces an existing label if it has the same key' do - select_label_and_refresh @new_label_same_scope - - labels_block = page.find(%q([data-qa-selector="labels_block"])) - - expect(labels_block).to have_content(@new_label_same_scope) - expect(labels_block).not_to have_content(@initial_label) - expect(page).to have_content("added #{@new_label_same_scope}") - expect(page).to have_content("and removed #{@initial_label}") -end - -it 'keeps both scoped label when adding a label with a different key' do - select_label_and_refresh @new_label_different_scope - - labels_block = page.find(%q([data-qa-selector="labels_block"])) - - expect(labels_blocks).to have_content(@new_label_different_scope) - expect(labels_blocks).to have_content(@initial_label) - expect(page).to have_content("added #{@new_label_different_scope}") - expect(page).to have_content("added #{@initial_label}") -end - -def select_label_and_refresh(label) - page.find('.block.labels .edit-link').click - page.find('.dropdown-menu-labels .dropdown-input-field').send_keys [label, :enter] - page.find('#content-body').click - page.refresh -end -``` - -First, we remove the duplication of strings by defining the global variables `@initial_label`, `@new_label_same_scope` and `@new_label_different_scope` in the `before` block, and by using them in the expectations. - -Then, by creating a reusable `select_label_and_refresh` method, we remove the code duplication of this action, and later we can move this method to a Page Object class that will be created for easier maintenance purposes. - -Notice that the reusable method is created at the bottom of the file. This helps readability, -where reading the code should be similar to reading a newspaper: - -- High-level information is at the top, like the title and summary of the news. -- Low level, or more specific information, is at the bottom. - -### 5. Tests' pre-conditions using resources and Page Objects - -In this section, we will address the previously mentioned subject of creating the application state for the tests, using the `before :context` and `before` blocks, together with resources and Page Objects. - -#### `before :context` - -A pre-condition for the entire test suite is defined in the `before :context` block. - -> For our test suite, due to the need of the tests being completely independent of each other, we won't use the `before :context` block. The `before :context` block would make the tests dependent on each other because the first test changes the label of the issue, and the second one depends on the `'animal::fox'` label being set. - -TIP: **Tip:** In case of a test suite with only one `it` block it's ok to use only the `before` block (see below) with all the test's pre-conditions. - -#### `before` - -As the pre-conditions for our test suite, the things that needs to happen before each test starts are: - -- The user logging in; -- A premium license already being set; -- A project being created with an issue and labels already set; -- The issue page being opened with only one scoped label applied to it. - -> When running end-to-end tests as part of the GitLab's continuous integration process [a license is already set as an environment variable](https://gitlab.com/gitlab-org/gitlab/blob/1a60d926740db10e3b5724713285780a4f470531/qa/qa/ee/strategy.rb#L20). For running tests locally you can set up such license by following the document [what tests can be run?](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/what_tests_can_be_run.md), based on the [supported GitLab environment variables](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/what_tests_can_be_run.md#supported-gitlab-environment-variables). - -#### Implementation - -In the following code we will focus only on the test suite's pre-conditions: - -```ruby -module QA - context 'Plan' do - describe 'Editing scoped labels on issues' do - before do - Runtime::Browser.visit(:gitlab, Page::Main::Login) - Page::Main::Login.perform(&:sign_in_using_credentials) - - @initial_label = 'animal::fox' - @new_label_same_scope = 'animal::dolphin' - @new_label_different_scope = 'plant::orchid' - - issue = Resource::Issue.fabricate_via_api! do |issue| - issue.title = 'Issue to test the scoped labels' - issue.labels = [@initial_label] - end - - [@new_label_same_scope, @new_label_different_scope].each do |label| - Resource::Label.fabricate_via_api! do |l| - l.project = issue.project - l.title = label - end - end - - issue.visit! - end - - it 'replaces an existing label if it has the same key' do - ... - end - - it 'keeps both scoped labels when adding a label with a different key' do - ... - end - - def select_label_and_refresh(label) - ... - end - end - end -end -``` - -In the `before` block we create all the application state needed for the tests to run. We do that by using the `Runtime::Browser.visit` method to go to the login page, by performing a `sign_in_using_credentials` from the `Login` Page Object, by fabricating resources via APIs (`issue`, and `Resource::Label`), and by using the `issue.visit!` to visit the issue page. - -> A project is created in the background by creating the `issue` resource. -> -> When creating the [Resources](resources.md), notice that when calling the `fabricate_via_api` method, we pass some attribute:values, like `title`, and `labels` for the `issue` resource; and `project` and `title` for the `label` resource. -> -> What's important to understand here is that by creating the application state mostly using the public APIs we save a lot of time in the test suite setup stage. -> -> Soon we will cover the use of the already existing resources' methods and the creation of your own `fabricate_via_api` methods for resources where this is still not available, but first, let's optimize our implementation. - -### 6. Optimization - -As already mentioned in the [best practices](best_practices.md) document, end-to-end tests are very costly in terms of execution time, and it's our responsibility as software engineers to ensure that we optimize them as much as possible. - -> Note that end-to-end tests are slow to run and so they can have several actions and assertions in a single test, which helps us get feedback from the tests sooner. In comparison, unit tests are much faster to run and can exercise every little piece of the application in isolation, and so they usually have only one assertion per test. - -Some improvements that we could make in our test suite to optimize its time to run are: - -1. Having a single test case (an `it` block) that exercises both scenarios to avoid "wasting" time in the tests' pre-conditions, instead of having two different test cases. -1. Making the selection of labels more performant by allowing for the selection of more than one label in the same reusable method. - -Let's look at a suggestion that addresses the above points, one by one: - -```ruby -module QA - context 'Plan' do - describe 'Editing scoped labels on issues' do - before do - ... - end - - it 'correctly applies scoped labels depending on if they are from the same or a different scope' do - select_labels_and_refresh [@new_label_same_scope, @new_label_different_scope] - - labels_block = page.all(%q([data-qa-selector="labels_block"])) - - expect(labels_block).to have_content(@new_label_same_scope) - expect(labels_block).to have_content(@new_label_different_scope) - expect(labels_block).not_to have_content(@initial_label) - expect(page).to have_content("added #{@initial_label}") - expect(page).to have_content("added #{@new_label_same_scope} #{@new_label_different_scope} labels and removed #{@initial_label}") - end - - def select_labels_and_refresh(labels) - find('.block.labels .edit-link').click - labels.each do |label| - find('.dropdown-menu-labels .dropdown-input-field').send_keys [label, :enter] - end - find('#content-body').click - refresh - end - end - end - end -``` - -To address point 1, we changed the test implementation from two `it` blocks into a single one that exercises both scenarios. Now the new test description is: `'correctly applies the scoped labels depending if they are from the same or a different scope'`. It's a long description, but it describes well what the test does. - -> Notice that the implementation of the new and unique `it` block had to change a little bit. Below we describe in details what it does. - -1. It selects two scoped labels simultaneously, one from the same scope of the one already applied in the issue during the setup phase (in the `before` block), and another one from a different scope. -1. It asserts that the correct labels are visible in the `labels_block`, and that the labels were correctly added and removed; -1. Finally, the `select_label_and_refresh` method is changed to `select_labels_and_refresh`, which accepts an array of labels instead of a single label, and it iterates on them for faster label selection (this is what is used in step 1 explained above.) - -### 7. Resources - -**Note:** When writing this document, some code that is now merged to master was not implemented yet, but we left them here for the readers to understand the whole process of end-to-end test creation. - -You can think of [Resources](resources.md) as anything that can be created on GitLab CE or EE, either through the GUI, the API, or the CLI. - -With that in mind, resources can be a project, an epic, an issue, a label, a commit, etc. - -As you saw in the tests' pre-conditions and the optimization sections, we're already creating some of these resources. We are doing that by calling the `fabricate_via_api!` method. - -> We could be using the `fabricate!` method instead, which would use the `fabricate_via_api!` method if it exists, and fallback to GUI fabrication otherwise, but we recommend being explicit to make it clear what the test does. Also, we always recommend fabricating resources via API since this makes tests faster and more reliable. - -For our test suite example, the resources that we need to create don't have the necessary code for the `fabricate_via_api!` method to correctly work (e.g., the issue and label resources), so we will have to create them. - -#### Implementation - -In the following we describe the changes needed in each of the resource files mentioned above. - -**Issue resource** - -Now, let's make it possible to create an issue resource through the API. - -First, in the [issue resource](https://gitlab.com/gitlab-org/gitlab/blob/d3584e80b4236acdf393d815d604801573af72cc/qa/qa/resource/issue.rb), let's expose its id and labels attributes. - -Add the following `attribute :id` and `attribute :labels` right above the [`attribute :title`](https://gitlab.com/gitlab-org/gitlab/blob/d3584e80b4236acdf393d815d604801573af72cc/qa/qa/resource/issue.rb#L15). - -> This line is needed to allow for the issue fabrication, and for labels to be automatically added to the issue when fabricating it via API. -> -> We add the attributes above the existing attribute to keep them alphabetically organized. - -Then, let's initialize an instance variable for labels to allow an empty array as default value when such information is not passed during the resource fabrication, since this optional. [Between the attributes and the `fabricate!` method](https://gitlab.com/gitlab-org/gitlab/blob/1a1f1408728f19b2aa15887cd20bddab7e70c8bd/qa/qa/resource/issue.rb#L18), add the following: - -```ruby -def initialize - @labels = [] -end -``` - -Next, add the following code right below the [`fabricate!`](https://gitlab.com/gitlab-org/gitlab/blob/d3584e80b4236acdf393d815d604801573af72cc/qa/qa/resource/issue.rb#L27) method. - -```ruby -def api_get_path - "/projects/#{project.id}/issues/#{id}" -end - -def api_post_path - "/projects/#{project.id}/issues" -end - -def api_post_body - { - labels: labels, - title: title - } -end -``` - -By defining the `api_get_path` method, we allow the [`ApiFabricator`](https://gitlab.com/gitlab-org/gitlab/blob/master/qa/qa/resource/api_fabricator.rb) module to know which path to use to get a single issue. - -> This `GET` path can be found in the [public API documentation](../../../api/issues.md#single-issue). - -By defining the `api_post_path` method, we allow the [`ApiFabricator`](https://gitlab.com/gitlab-org/gitlab/blob/master/qa/qa/resource/api_fabricator.rb) module to know which path to use to create a new issue in a specific project. - -> This `POST` path can be found in the [public API documentation](../../../api/issues.md#new-issue). - -By defining the `api_post_body` method, we allow the [`ApiFabricator.api_post`](https://gitlab.com/gitlab-org/gitlab/blob/a9177ca1812bac57e2b2fa4560e1d5dd8ffac38b/qa/qa/resource/api_fabricator.rb#L68) method to know which data to send when making the `POST` request. - -> Notice that we pass both `labels` and `title` attributes in the `api_post_body`, where `labels` receives an array of labels, and [`title` is required](../../../api/issues.md#new-issue). Also, notice that we keep them alphabetically organized. - -**Label resource** - -Finally, let's make it possible to create label resources through the API. - -Add the following code right below the [`fabricate!`](https://gitlab.com/gitlab-org/gitlab/blob/a9177ca1812bac57e2b2fa4560e1d5dd8ffac38b/qa/qa/resource/label.rb#L36) method. - -```ruby -def resource_web_url(resource) - super -rescue ResourceURLMissingError - # this particular resource does not expose a web_url property -end - -def api_get_path - raise NotImplementedError, "The Labels API doesn't expose a single-resource endpoint so this method cannot be properly implemented." -end - -def api_post_path - "/projects/#{project.id}/labels" -end - -def api_post_body - { - color: @color, - name: @title - } -end -``` - -By defining the `resource_web_url(resource)` method, we override the one from the [`ApiFabricator`](https://gitlab.com/gitlab-org/gitlab/blob/master/qa/qa/resource/api_fabricator.rb#L44) module. We do that to avoid failing the test due to this particular resource not exposing a `web_url` property. - -By defining the `api_get_path` method, we **would** allow for the [`ApiFabricator`](https://gitlab.com/gitlab-org/gitlab/blob/master/qa/qa/resource/api_fabricator.rb) module to know which path to use to get a single label, but since there's no path available for that in the public API, we raise a `NotImplementedError` instead. - -By defining the `api_post_path` method, we allow for the [`ApiFabricator`](https://gitlab.com/gitlab-org/gitlab/blob/master/qa/qa/resource/api_fabricator.rb) module to know which path to use to create a new label in a specific project. - -By defining the `api_post_body` method, we allow for the [`ApiFabricator.api_post`](https://gitlab.com/gitlab-org/gitlab/blob/a9177ca1812bac57e2b2fa4560e1d5dd8ffac38b/qa/qa/resource/api_fabricator.rb#L68) method to know which data to send when making the `POST` request. - -> Notice that we pass both `color` and `name` attributes in the `api_post_body` since [those are required](../../../api/labels.md#create-a-new-label). Also, notice that we keep them alphabetically organized. - -### 8. Page Objects - -Page Objects are used in end-to-end tests for maintenance reasons, where a page's elements and methods are defined to be reused in any test. - -> Page Objects are auto-loaded in the [`qa/qa.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/qa/qa.rb) file and available in all the test files (`*_spec.rb`). - -Take a look at the [Page Objects](page_objects.md) documentation. - -Now, let's go back to our example. - -As you may have noticed, we are defining elements with CSS selectors and the `select_labels_and_refresh` method directly in the test file, and this is an anti-pattern since we need to better separate the responsibilities. - -To address this issue, we will move the implementation to Page Objects, and the test suite will only focus on the business rules that we are testing. - -#### Updates in the test file - -As in a test-driven development approach, let's start changing the test file even before the Page Object implementation is in place. - -Replace the code of the `it` block in the test file by the following: - -```ruby -module QA - context 'Plan' do - describe 'Editing scoped labels on issues' do - before do - ... - end - - it 'correctly applies scoped labels depending on if they are from the same or a different scope' do - Page::Project::Issue::Show.perform do |issue_page| - issue_page.select_labels_and_refresh [@new_label_same_scope, @new_label_different_scope] - - expect(page).to have_content("added #{@initial_label}") - expect(page).to have_content("added #{@new_label_same_scope} #{@new_label_different_scope} labels and removed #{@initial_label}") - expect(issue_page.text_of_labels_block).to have_content(@new_label_same_scope) - expect(issue_page.text_of_labels_block).to have_content(@new_label_different_scope) - expect(issue_page.text_of_labels_block).not_to have_content(@initial_label) - end - end - end - end -end -``` - -Notice that `select_labels_and_refresh` is now a method from the issue Page Object (which is not yet implemented), and that we verify the labels' text by using `text_of_labels_block`, instead of via the `labels_block` element. The `text_of_labels_block` method will also be implemented in the issue Page Object. - -Let's now update the Issue Page Object. - -#### Updates in the Issue Page Object - -> Page Objects are located in the `qa/qa/page/` directory, and its sub-directories. - -The file we will have to change is the [Issue Page Object](https://gitlab.com/gitlab-org/gitlab/blob/master/qa/qa/page/project/issue/show.rb). - -First, add the following code right below the definition of an already implemented view (keep in mind that view's definitions and their elements should be alphabetically ordered): - -```ruby -view 'app/helpers/dropdowns_helper.rb' do - element :dropdown_input_field -end - -view 'app/views/shared/issuable/_sidebar.html.haml' do - element :dropdown_menu_labels - element :edit_link_labels - element :labels_block -end -``` - -Similarly to what we did before, let's first change the Page Object even without the elements being defined in the view (`_sidebar.html.haml`) and the `dropdowns_helper.rb` files, and later we will update them by adding the appropriate CSS selectors. - -Now, let's implement the methods `select_labels_and_refresh` and `text_of_labels_block`. - -Somewhere between the definition of the views and the private methods, add the following snippet of code (these should also be alphabetically ordered for organization reasons): - -```ruby -def select_labels_and_refresh(labels) - click_element(:edit_link_labels) - labels.each do |label| - within_element(:dropdown_menu_labels, text: label) do - send_keys_to_element(:dropdown_input_field, [label, :enter]) - end - end - click_body - labels.each do |label| - has_element?(:labels_block, text: label) - end - refresh -end - -def text_of_labels_block - find_element(:labels_block) -end -``` - -##### Details of `select_labels_and_refresh` - -Notice that we have not only moved the `select_labels_and_refresh` method, but we have also changed its implementation to: - -1. Click the `:edit_link_labels` element previously defined, instead of using `find('.block.labels .edit-link').click` -1. Use `within_element(:dropdown_menu_labels, text: label)`, and inside of it, we call `send_keys_to_element(:dropdown_input_field, [label, :enter])`, which is a method that we will implement in the `QA::Page::Base` class to replace `find('.dropdown-menu-labels .dropdown-input-field').send_keys [label, :enter]` -1. Use `click_body` after iterating on each label, instead of using `find('#content-body').click` -1. Iterate on every label again, and then we use `has_element?(:labels_block, text: label)` after clicking the page body (which applies the labels), and before refreshing the page, to avoid test flakiness due to refreshing too fast. - -##### Details of `text_of_labels_block` - -The `text_of_labels_block` method is a simple method that returns the `:labels_block` element (`find_element(:labels_block)`). - -#### Updates in the view (*.html.haml) and `dropdowns_helper.rb` files - -Now let's change the view and the `dropdowns_helper` files to add the selectors that relate to the [Page Objects](page_objects.md). - -In [`app/views/shared/issuable/_sidebar.html.haml:105`](https://gitlab.com/gitlab-org/gitlab/blob/7ca12defc7a965987b162a6ebef302f95dc8867f/app/views/shared/issuable/_sidebar.html.haml#L105), add a `data: { qa_selector: 'edit_link_labels' }` data attribute. - -The code should look like this: - -```haml -= link_to _('Edit'), '#', class: 'js-sidebar-dropdown-toggle edit-link float-right', data: { qa_selector: 'edit_link_labels' } -``` - -In the same file, on [line 121](https://gitlab.com/gitlab-org/gitlab/blob/7ca12defc7a965987b162a6ebef302f95dc8867f/app/views/shared/issuable/_sidebar.html.haml#L121), add a `data: { qa_selector: 'dropdown_menu_labels' }` data attribute. - -The code should look like this: - -```haml -.dropdown-menu.dropdown-select.dropdown-menu-paging.dropdown-menu-labels.dropdown-menu-selectable.dropdown-extended-height{ data: { qa_selector: 'dropdown_menu_labels' } } -``` - -In [`app/helpers/dropdowns_helper.rb:94`](https://gitlab.com/gitlab-org/gitlab/blob/7ca12defc7a965987b162a6ebef302f95dc8867f/app/helpers/dropdowns_helper.rb#L94), add a `data: { qa_selector: 'dropdown_input_field' }` data attribute. - -The code should look like this: - -```ruby -filter_output = search_field_tag search_id, nil, class: "dropdown-input-field", placeholder: placeholder, autocomplete: 'off', data: { qa_selector: 'dropdown_input_field' } -``` - -> `data-qa-*` data attributes and CSS classes starting with `qa-` are used solely for the purpose of QA and testing. -> By defining these, we add **testability** to the application. -> -> When defining a data attribute like: `qa_selector: 'labels_block'`, it should match the element definition: `element :labels_block`. We use a [sanity test](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/doc/development/testing_guide/end_to_end/page_objects.md#how-did-we-solve-fragile-tests-problem) to check that defined elements have their respective selectors in the specified views. - -#### Updates in the `QA::Page::Base` class - -The last thing that we have to do is to update `QA::Page::Base` class to add the `send_keys_to_element` method on it. - -Add the following snippet of code somewhere where class methods are defined (remember to organize methods alphabetically, and if you see a place where this standard is not being followed, it would be helpful if you could rearrange it): - -```ruby -def send_keys_to_element(name, keys) - find_element(name).send_keys(keys) -end -``` - -This method receives an element (`name`) and the `keys` that it will send to that element, and the keys are an array that can receive strings, or "special" keys, like `:enter`. - -As you might remember, in the Issue Page Object we call this method like this: `send_keys_to_element(:dropdown_input_field, [label, :enter])`. - -With that, you should be able to start writing end-to-end tests yourself. *Congratulations!* diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index f8dc3366904..b7c93d205a3 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -14,3 +14,5 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | `:reliable` | The test has been [promoted to a reliable test](https://about.gitlab.com/handbook/engineering/quality/guidelines/reliable-tests/#promoting-an-existing-test-to-reliable) meaning it passes consistently in all pipelines, including merge requests. | | `:requires_admin` | The test requires an admin account. Tests with the tag are excluded when run against Canary and Production environments. | | `:runner` | The test depends on and will set up a GitLab Runner instance, typically to run a pipeline. | +| `:gitaly_ha` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | +| `:skip_live_env` | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. | diff --git a/doc/development/testing_guide/end_to_end/style_guide.md b/doc/development/testing_guide/end_to_end/style_guide.md index 7f4616f394b..9c02af12d5d 100644 --- a/doc/development/testing_guide/end_to_end/style_guide.md +++ b/doc/development/testing_guide/end_to_end/style_guide.md @@ -66,6 +66,7 @@ We follow a simple formula roughly based on hungarian notation. - `_placeholder`: a temporary element that appears while content is loading. For example, the elements that are displayed instead of discussions while the discussions are being fetched. - `_radio` - `_tab` + - `_menu_item` *Note: If none of the listed types are suitable, please open a merge request to add an appropriate type to the list.* diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md index 1e53e92fad5..6c1b06ce59a 100644 --- a/doc/development/testing_guide/flaky_tests.md +++ b/doc/development/testing_guide/flaky_tests.md @@ -40,7 +40,7 @@ Quarantined tests are run on the CI in dedicated jobs that are allowed to fail: ## Automatic retries and flaky tests detection -On our CI, we use [rspec-retry](https://github.com/NoRedInk/rspec-retry) to automatically retry a failing example a few +On our CI, we use [RSpec::Retry](https://github.com/NoRedInk/rspec-retry) to automatically retry a failing example a few times (see [`spec/spec_helper.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/spec_helper.rb) for the precise retries count). We also use a home-made `RspecFlaky::Listener` listener which records flaky diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 2a3fcf122a6..52d538c7159 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -5,7 +5,7 @@ at GitLab. We use Karma with Jasmine and Jest for JavaScript unit and integratio and RSpec feature tests with Capybara for e2e (end-to-end) integration testing. Unit and feature tests need to be written for all new features. -Most of the time, you should use [RSpec] for your feature tests. +Most of the time, you should use [RSpec](https://github.com/rspec/rspec-rails#feature-specs) for your feature tests. Regression tests should be written for bug fixes to prevent them from recurring in the future. @@ -15,7 +15,7 @@ information on general testing practices at GitLab. ## Vue.js testing -If you are looking for a guide on Vue component testing, you can jump right away to this [section][vue-test]. +If you are looking for a guide on Vue component testing, you can jump right away to this [section](../fe_guide/vue.md#testing-vue-components). ## Jest @@ -30,7 +30,7 @@ Jest tests can be found in `/spec/frontend` and `/ee/spec/frontend` in EE. ## Karma test suite -While GitLab is switching over to [Jest][jest] you'll still find Karma tests in our application. [Karma][karma] is a test runner which uses [Jasmine] as its test +While GitLab is switching over to [Jest](https://jestjs.io) you'll still find Karma tests in our application. [Karma](http://karma-runner.github.io/) is a test runner which uses [Jasmine](https://jasmine.github.io/) as its test framework. Jest also uses Jasmine as foundation, that's why it's looking quite similar. Karma tests live in `spec/javascripts/` and `/ee/spec/javascripts` in EE. @@ -549,7 +549,8 @@ TBU Jasmine provides stubbing and mocking capabilities. There are some subtle differences in how to use it within Karma and Jest. -Stubs or spies are often used synonymously. In Jest it's quite easy thanks to the `.spyOn` method. [Official docs][jestspy] +Stubs or spies are often used synonymously. In Jest it's quite easy thanks to the `.spyOn` method. +[Official docs](https://jestjs.io/docs/en/jest-object#jestspyonobject-methodname) The more challenging part are mocks, which can be used for functions or even dependencies. ### Manual module mocks @@ -637,12 +638,12 @@ Karma allows something similar, but it's way more costly. Running Karma with `yarn run karma-start` will compile the JavaScript assets and run a server at `http://localhost:9876/` where it will automatically -run the tests on any browser which connects to it. You can enter that url on +run the tests on any browser which connects to it. You can enter that URL on multiple browsers at once to have it run the tests on each in parallel. While Karma is running, any changes you make will instantly trigger a recompile and retest of the **entire test suite**, so you can see instantly if you've broken -a test with your changes. You can use [Jasmine focused][jasmine-focus] or +a test with your changes. You can use [Jasmine focused](https://jasmine.github.io/2.5/focused_specs.html) or excluded tests (with `fdescribe` or `xdescribe`) to get Karma to run only the tests you want while you're working on a specific feature, but make sure to remove these directives when you commit your code. @@ -831,43 +832,6 @@ testAction( Check an example in [spec/javascripts/ide/stores/actions_spec.jsspec/javascripts/ide/stores/actions_spec.js](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/javascripts/ide/stores/actions_spec.js). -### Vue Helper: `mountComponent` - -To make mounting a Vue component easier and more readable, we have a few helpers available in `spec/helpers/vue_mount_component_helper`: - -- `createComponentWithStore` -- `mountComponentWithStore` - -Examples of usage: - -```javascript -beforeEach(() => { - vm = createComponentWithStore(Component, store); - - vm.$store.state.currentBranchId = 'master'; - - vm.$mount(); -}); -``` - -```javascript -beforeEach(() => { - vm = mountComponentWithStore(Component, { - el: '#dummy-element', - store, - props: { badge }, - }); -}); -``` - -Don't forget to clean up: - -```javascript -afterEach(() => { - vm.$destroy(); -}); -``` - ### Wait until axios requests finish The axios utils mock module located in `spec/frontend/mocks/ce/lib/utils/axios_utils.js` contains two helper methods for Jest tests that spawn HTTP requests. @@ -906,13 +870,3 @@ You can download any older version of Firefox from the releases FTP server, <htt --- [Return to Testing documentation](index.md) - -<!-- URL References --> - -[jasmine-focus]: https://jasmine.github.io/2.5/focused_specs.html -[karma]: http://karma-runner.github.io/ -[vue-test]: ../fe_guide/vue.md#testing-vue-components -[rspec]: https://github.com/rspec/rspec-rails#feature-specs -[jasmine]: https://jasmine.github.io/ -[jest]: https://jestjs.io -[jestspy]: https://jestjs.io/docs/en/jest-object#jestspyonobject-methodname diff --git a/doc/development/testing_guide/index.md b/doc/development/testing_guide/index.md index 25da72b6b6a..f0fb06910f8 100644 --- a/doc/development/testing_guide/index.md +++ b/doc/development/testing_guide/index.md @@ -4,15 +4,15 @@ This document describes various guidelines and best practices for automated testing of the GitLab project. It is meant to be an _extension_ of the [thoughtbot testing -styleguide](https://github.com/thoughtbot/guides/tree/master/style/testing). If +style guide](https://github.com/thoughtbot/guides/tree/master/style/testing). If this guide defines a rule that contradicts the thoughtbot guide, this guide takes precedence. Some guidelines may be repeated verbatim to stress their importance. ## Overview -GitLab is built on top of [Ruby on Rails](https://rubyonrails.org/), and we're using [RSpec] for all -the backend tests, with [Capybara] for end-to-end integration testing. +GitLab is built on top of [Ruby on Rails](https://rubyonrails.org/), and we're using [RSpec](https://github.com/rspec/rspec-rails#feature-specs) for all +the backend tests, with [Capybara](https://github.com/teamcapybara/capybara) for end-to-end integration testing. On the frontend side, we're using [Jest](https://jestjs.io/) and [Karma](http://karma-runner.github.io/)/[Jasmine](https://jasmine.github.io/) for JavaScript unit and integration testing. @@ -58,14 +58,10 @@ Everything you should know about how to test Rake tasks. ## [End-to-end tests](end_to_end/index.md) Everything you should know about how to run end-to-end tests using -[GitLab QA][gitlab-qa] testing framework. +[GitLab QA](ttps://gitlab.com/gitlab-org/gitlab-qa) testing framework. ## [Migrations tests](testing_migrations_guide.md) Everything you should know about how to test migrations. [Return to Development documentation](../README.md) - -[RSpec]: https://github.com/rspec/rspec-rails#feature-specs -[Capybara]: https://github.com/teamcapybara/capybara -[gitlab-qa]: https://gitlab.com/gitlab-org/gitlab-qa diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 9eb5d5add8a..58acf937d77 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -102,10 +102,10 @@ subgraph "CNG-mirror pipeline" ### Auto-stopping of Review Apps Review Apps are automatically stopped 2 days after the last deployment thanks to -the [Environment auto-stop](../../ci/environments.md#environments-auto-stop) feature. +the [Environment auto-stop](../../ci/environments/index.md#environments-auto-stop) feature. If you need your Review App to stay up for a longer time, you can -[pin its environment](../../ci/environments.md#auto-stop-example) or retry the +[pin its environment](../../ci/environments/index.md#auto-stop-example) or retry the `review-deploy` job to update the "latest deployed at" time. The `review-cleanup` job that automatically runs in scheduled @@ -153,7 +153,13 @@ used by the `review-deploy` and `review-stop` jobs. ### Get access to the GCP Review Apps cluster You need to [open an access request (internal link)](https://gitlab.com/gitlab-com/access-requests/issues/new) -for the `gcp-review-apps-sg` GCP group. +for the `gcp-review-apps-sg` GCP group. In order to join a group, you must specify the desired GCP role in your access request. +The role is what will grant you specific permissions in order to engage with Review App containers. + +Here are some permissions you may want to have, and the roles that grant them: + +- `container.pods.getLogs` - Required to [retrieve pod logs](#dig-into-a-pods-logs). Granted by [Viewer (`roles/viewer`)](https://cloud.google.com/iam/docs/understanding-roles#kubernetes-engine-roles). +- `container.pods.exec` - Required to [run a Rails console](#run-a-rails-console). Granted by [Kubernetes Engine Developer (`roles/container.developer`)](https://cloud.google.com/iam/docs/understanding-roles#kubernetes-engine-roles). ### Log into my Review App @@ -175,7 +181,7 @@ secure note named `gitlab-{ce,ee} Review App's root password`. ### Run a Rails console -1. Make sure you [have access to the cluster](#get-access-to-the-gcp-review-apps-cluster) first. +1. Make sure you [have access to the cluster](#get-access-to-the-gcp-review-apps-cluster) and the `container.pods.exec` permission first. 1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps), e.g. `review-qa-raise-e-12chm0`. 1. Find and open the `task-runner` Deployment, e.g. `review-qa-raise-e-12chm0-task-runner`. @@ -191,7 +197,7 @@ secure note named `gitlab-{ce,ee} Review App's root password`. ### Dig into a Pod's logs -1. Make sure you [have access to the cluster](#get-access-to-the-gcp-review-apps-cluster) first. +1. Make sure you [have access to the cluster](#get-access-to-the-gcp-review-apps-cluster) and the `container.pods.getLogs` permission first. 1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps), e.g. `review-qa-raise-e-12chm0`. 1. Find and open the `migrations` Deployment, e.g. @@ -209,6 +215,31 @@ Leading indicators may be health check failures leading to restarts or majority The [Review Apps Overview dashboard](https://app.google.stackdriver.com/dashboards/6798952013815386466?project=gitlab-review-apps&timeDomain=1d) aids in identifying load spikes on the cluster, and if nodes are problematic or the entire cluster is trending towards unhealthy. +### Release failed with `ImagePullBackOff` + +**Potential cause:** + +If you see an `ImagePullBackoff` status, check for a missing Docker image. + +**Where to look for further debugging:** + +To check that the Docker images were created, run the following Docker command: + +```shell +`DOCKER_CLI_EXPERIMENTAL=enabled docker manifest repository:tag` +``` + +The output of this command indicates if the Docker image exists. For example: + +```shell +DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee:39467-allow-a-release-s-associated-milestones-to-be-edited-thro +``` + +If the Docker image does not exist: + +- Verify the `image.repository` and `image.tag` options in the `helm upgrade --install` command match the repository names used by CNG-mirror pipeline. +- Look further in the corresponding downstream CNG-mirror pipeline in `review-build-cng` job. + ### Node count is always increasing (i.e. never stabilizing or decreasing) **Potential cause:** diff --git a/doc/development/testing_guide/testing_levels.md b/doc/development/testing_guide/testing_levels.md index 295aa6609a8..9285a910ecf 100644 --- a/doc/development/testing_guide/testing_levels.md +++ b/doc/development/testing_guide/testing_levels.md @@ -312,9 +312,7 @@ of a controller test. Testing a fat controller usually involves a lot of stubbin controller.instance_variable_set(:@user, user) ``` -and use methods which are deprecated in Rails 5 ([#23768]). - -[#23768]: https://gitlab.com/gitlab-org/gitlab/-/issues/16260 +and use methods which are deprecated in Rails 5 ([#23768](https://gitlab.com/gitlab-org/gitlab/-/issues/16260)). ### About Karma @@ -356,7 +354,7 @@ possible). | Tests path | Testing engine | Notes | | ---------- | -------------- | ----- | -| `spec/features/` | [Capybara] + [RSpec] | If your test has the `:js` metadata, the browser driver will be [Poltergeist], otherwise it's using [RackTest]. | +| `spec/features/` | [Capybara](https://github.com/teamcapybara/capybara) + [RSpec](https://github.com/rspec/rspec-rails#feature-specs) | If your test has the `:js` metadata, the browser driver will be [Poltergeist](https://github.com/teamcapybara/capybara#poltergeist), otherwise it's using [RackTest](https://github.com/teamcapybara/capybara#racktest). | ### Frontend feature tests @@ -460,9 +458,6 @@ The reasons why we should follow these best practices are as follows: of tests). This is slower than transactions, however, so we want to use truncation only when necessary. -[Poltergeist]: https://github.com/teamcapybara/capybara#poltergeist -[RackTest]: https://github.com/teamcapybara/capybara#racktest - ## Black-box tests at the system level, aka end-to-end tests Formal definitions: @@ -470,11 +465,11 @@ Formal definitions: - <https://en.wikipedia.org/wiki/System_testing> - <https://en.wikipedia.org/wiki/Black-box_testing> -GitLab consists of [multiple pieces] such as [GitLab Shell], [GitLab Workhorse], -[Gitaly], [GitLab Pages], [GitLab Runner], and GitLab Rails. All theses pieces -are configured and packaged by [GitLab Omnibus]. +GitLab consists of [multiple pieces](../architecture.md#components) such as [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell), [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse), +[Gitaly](https://gitlab.com/gitlab-org/gitaly), [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages), [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner), and GitLab Rails. All theses pieces +are configured and packaged by [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab). -The QA framework and instance-level scenarios are [part of GitLab Rails] so that +The QA framework and instance-level scenarios are [part of GitLab Rails](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa) so that they're always in-sync with the codebase (especially the views). Note that: @@ -483,11 +478,11 @@ Note that: - data needed for the tests can only be created using the GUI or the API - expectations can only be made against the browser page and API responses -Every new feature should come with a [test plan]. +Every new feature should come with a [test plan](https://gitlab.com/gitlab-org/gitlab/tree/master/.gitlab/issue_templates/Test%20plan.md). | Tests path | Testing engine | Notes | | ---------- | -------------- | ----- | -| `qa/qa/specs/features/` | [Capybara] + [RSpec] + Custom QA framework | Tests should be placed under their corresponding [Product category] | +| `qa/qa/specs/features/` | [Capybara](https://github.com/teamcapybara/capybara) + [RSpec](https://github.com/rspec/rspec-rails#feature-specs) + Custom QA framework | Tests should be placed under their corresponding [Product category](https://about.gitlab.com/handbook/product/categories/) | > See [end-to-end tests](end_to_end/index.md) for more information. @@ -495,17 +490,6 @@ Note that `qa/spec` contains unit tests of the QA framework itself, not to be confused with the application's [unit tests](#unit-tests) or [end-to-end tests](#black-box-tests-at-the-system-level-aka-end-to-end-tests). -[multiple pieces]: ../architecture.md#components -[GitLab Shell]: https://gitlab.com/gitlab-org/gitlab-shell -[GitLab Workhorse]: https://gitlab.com/gitlab-org/gitlab-workhorse -[Gitaly]: https://gitlab.com/gitlab-org/gitaly -[GitLab Pages]: https://gitlab.com/gitlab-org/gitlab-pages -[GitLab Runner]: https://gitlab.com/gitlab-org/gitlab-runner -[GitLab Omnibus]: https://gitlab.com/gitlab-org/omnibus-gitlab -[part of GitLab Rails]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa -[test plan]: https://gitlab.com/gitlab-org/gitlab/tree/master/.gitlab/issue_templates/Test%20plan.md -[Product category]: https://about.gitlab.com/handbook/product/categories/ - ### Smoke tests Smoke tests are quick tests that may be run at any time (especially after the @@ -517,14 +501,11 @@ These tests run against the UI and ensure that basic functionality is working. ### GitLab QA orchestrator -[GitLab QA orchestrator] is a tool that allows to test that all these pieces +[GitLab QA orchestrator](https://gitlab.com/gitlab-org/gitlab-qa) is a tool that allows to test that all these pieces integrate well together by building a Docker image for a given version of GitLab Rails and running end-to-end tests (i.e. using Capybara) against it. -Learn more in the [GitLab QA orchestrator README][gitlab-qa-readme]. - -[GitLab QA orchestrator]: https://gitlab.com/gitlab-org/gitlab-qa -[gitlab-qa-readme]: https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md +Learn more in the [GitLab QA orchestrator README](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md). ## EE-specific tests @@ -538,7 +519,9 @@ trade-off: - Unit tests are usually cheap, and you should consider them like the basement of your house: you need them to be confident that your code is behaving correctly. However if you run only unit tests without integration / system - tests, you might [miss] the [big] / [picture] ! + tests, you might [miss](https://twitter.com/ThePracticalDev/status/850748070698651649) the + [big](https://twitter.com/timbray/status/822470746773409794) / + [picture](https://twitter.com/withzombies/status/829716565834752000) ! - Integration tests are a bit more expensive, but don't abuse them. A system test is often better than an integration test that is stubbing a lot of internals. - System tests are expensive (compared to unit tests), even more if they require @@ -546,8 +529,8 @@ trade-off: section. Another way to see it is to think about the "cost of tests", this is well -explained [in this article][tests-cost] and the basic idea is that the cost of a -test includes: +explained [in this article](https://medium.com/table-xi/high-cost-tests-and-high-value-tests-a86e27a54df#.2ulyh3a4e) +and the basic idea is that the cost of a test includes: - The time it takes to write the test - The time it takes to run the test every time the suite runs @@ -557,18 +540,11 @@ test includes: ### Frontend-related tests -There are cases where the behaviour you are testing is not worth the time spent +There are cases where the behavior you are testing is not worth the time spent running the full application, for example, if you are testing styling, animation, edge cases or small actions that don't involve the backend, you should write an integration test using Jasmine. -[miss]: https://twitter.com/ThePracticalDev/status/850748070698651649 -[big]: https://twitter.com/timbray/status/822470746773409794 -[picture]: https://twitter.com/withzombies/status/829716565834752000 -[tests-cost]: https://medium.com/table-xi/high-cost-tests-and-high-value-tests-a86e27a54df#.2ulyh3a4e -[RSpec]: https://github.com/rspec/rspec-rails#feature-specs -[Capybara]: https://github.com/teamcapybara/capybara - --- [Return to Testing documentation](index.md) diff --git a/doc/development/uploads.md b/doc/development/uploads.md index fbdb15d82e1..1dd6ab5496a 100644 --- a/doc/development/uploads.md +++ b/doc/development/uploads.md @@ -11,7 +11,7 @@ The following graph explains machine boundaries in a scalable GitLab installatio ```mermaid graph TB subgraph "load balancers" - LB(HA Proxy) + LB(Proxy) end subgraph "Shared storage" diff --git a/doc/development/value_stream_analytics.md b/doc/development/value_stream_analytics.md index 3ceb523ab73..69b07eb7c86 100644 --- a/doc/development/value_stream_analytics.md +++ b/doc/development/value_stream_analytics.md @@ -185,7 +185,7 @@ Currently supported parents: The [original implementation](https://gitlab.com/gitlab-org/gitlab/issues/847) of value stream analytics defined 7 stages. These stages are always available for each parent, however altering these stages is not possible. -To make things efficient and reduce the number of records created, the default stages are expressed as in-memory objects (not persisted). When the user creates a custom stage for the first time, all the stages will be persisted. This behaviour is implemented in the value stream analytics service objects. +To make things efficient and reduce the number of records created, the default stages are expressed as in-memory objects (not persisted). When the user creates a custom stage for the first time, all the stages will be persisted. This behavior is implemented in the value stream analytics service objects. The reason for this was that we'd like to add the abilities to hide and order stages later on. diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md index 1413c782c5d..f6c78e51299 100644 --- a/doc/development/verifying_database_capabilities.md +++ b/doc/development/verifying_database_capabilities.md @@ -2,7 +2,7 @@ Sometimes certain bits of code may only work on a certain database version. While we try to avoid such code as much as possible sometimes it is -necessary to add database (version) specific behaviour. +necessary to add database (version) specific behavior. To facilitate this we have the following methods that you can use: @@ -12,7 +12,7 @@ To facilitate this we have the following methods that you can use: This allows you to write code such as: ```ruby -if Gitlab::Database.version.to_f >= 9.6 +if Gitlab::Database.version.to_f >= 11.7 run_really_fast_query else run_fast_query diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index 9ece6eff41e..8ea9f70fc7a 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -5,38 +5,6 @@ GitLab offline, others do require a downtime period. This guide describes various operations, their impact, and how to perform them without requiring downtime. -## Adding Columns - -You can safely add a new column to an existing table as long as it does **not** -have a default value. For example, this query would not require downtime: - -```sql -ALTER TABLE projects ADD COLUMN random_value int; -``` - -Add a column _with_ a default however does require downtime. For example, -consider this query: - -```sql -ALTER TABLE projects ADD COLUMN random_value int DEFAULT 42; -``` - -This requires updating every single row in the `projects` table so that -`random_value` is set to `42` by default. This requires updating all rows and -indexes in a table. This in turn acquires enough locks on the table for it to -effectively block any other queries. - -Adding a column with a default value _can_ be done without requiring downtime -when using the migration helper method -`Gitlab::Database::MigrationHelpers#add_column_with_default`. This method works -similar to `add_column` except it updates existing rows in batches without -blocking access to the table being modified. See ["Adding Columns With Default -Values"](migration_style_guide.md#adding-columns-with-default-values) for more -information on how to use this method. - -Note that usage of `add_column_with_default` with `allow_null: false` to also add -a `NOT NULL` constraint is [discouraged](https://gitlab.com/gitlab-org/gitlab/issues/38060). - ## Dropping Columns Removing columns is tricky because running GitLab processes may still be using @@ -171,8 +139,39 @@ Adding or removing a NOT NULL clause (or another constraint) can typically be done without requiring downtime. However, this does require that any application changes are deployed _first_. Thus, changing the constraints of a column should happen in a post-deployment migration. -NOTE: Avoid using `change_column` as it produces inefficient query because it re-defines -the whole column type. For example, to add a NOT NULL constraint, prefer `change_column_null` + +NOTE: Avoid using `change_column` as it produces an inefficient query because it re-defines +the whole column type. + +To add a NOT NULL constraint, use the `add_not_null_constraint` migration helper: + +```ruby +# A post-deployment migration in db/post_migrate +class AddNotNull < ActiveRecord::Migration[4.2] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_not_null_constraint :users, :username + end + + def down + remove_not_null_constraint :users, :username + end +end +``` + +If the column to be updated requires cleaning first (e.g. there are `NULL` values), you should: + +1. Add the `NOT NULL` constraint with `validate: false` + + `add_not_null_constraint :users, :username, validate: false` + +1. Clean up the data with a data migration +1. Validate the `NOT NULL` constraint with a followup migration + + `validate_not_null_constraint :users, :username` ## Changing Column Types diff --git a/doc/development/windows.md b/doc/development/windows.md new file mode 100644 index 00000000000..b5309002222 --- /dev/null +++ b/doc/development/windows.md @@ -0,0 +1,139 @@ +--- +type: reference, howto +--- + +# Windows Development + +There are times in development where a Windows development machine is needed. +This is a guide for how to get a Windows development virtual machine on Google Cloud Platform +(GCP) with the same preinstalled tools as the GitLab shared Windows runners. + +## Why Windows in Google Cloud? + +Use of Microsoft Windows operating systems on company laptops is banned under GitLab's [Approved Operating Systems policy](https://about.gitlab.com/handbook/security/approved_os.html#windows). + +This can make it difficult to develop features for the Windows platforms. Using GCP will allow us to have a temporary Windows machine that can be removed once we're done with it. + +## Shared Windows runners + +You can use the shared Windows runners in the case that you don't need a full Windows development machine. +The [GitLab 12.7 Release Post](https://about.gitlab.com/releases/2020/01/22/gitlab-12-7-released/#windows-shared-runners-on-gitlabcom-beta) +and [Windows shared runner beta blog post](https://about.gitlab.com/blog/2020/01/21/windows-shared-runner-beta/#getting-started) both +outline quite a bit of useful information. + +To use the shared Windows runners add the following `tags` to relevant jobs in your `.gitlab-ci.yml` file: + +```yaml +tags: + - shared-windows + - windows + - windows-1809 +``` + +A list of software preinstalled on the Windows images is available at: [Preinstalled software](https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/gcp/windows-containers/blob/master/cookbooks/preinstalled-software/README.md). + +## GCP Windows image for development + +The [shared Windows GitLab +runners](https://about.gitlab.com/releases/2020/01/22/gitlab-12-7-released/#windows-shared-runners-on-gitlabcom-beta) +are built with [Packer](https://www.packer.io/). + +The Infrastructure as Code repository for building the Google Cloud images is available at: +[GitLab Google Cloud Platform Shared Runner Images](https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/gcp/windows-containers). + +### Build image + +There is a chance that your Google Cloud group may already have an image +built. Search the available images before you do the work to build your +own. + +Build a Google Cloud image with the above shared runners repo by doing the following: + +1. Install [Packer](https://www.packer.io/) (tested to work with version 1.5.1). +1. Install Packer Windows Update Provisioner. + 1. Clone the repository <https://github.com/rgl/packer-provisioner-windows-update> and `cd` into the cloned directory. + 1. Run the command `go build -o packer-provisioner-windows-update` (requires `go` to be installed). + 1. Verify `packer-provisioner-windows-update` is in the `PATH` environment variable. +1. Add all [required environment variables](https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/gcp/windows-containers/-/blob/master/packer.json#L2-10) + in the `packer.json` file to your environment (perhaps use [direnv](https://direnv.net/)). +1. Build the image by running the command: `packer build packer.json`. + +## How to use a Windows image in GCP + +1. In a web browser, go to <https://console.cloud.google.com/compute/images>. +1. Filter images by the name you used when creating image, `windows` is likely all you need to filter by. +1. Click the image's name. +1. Click the **CREATE INSTANCE** link. +1. Important: Change name to what you'd like as you can't change it later. +1. Optional: Change Region to be closest to you as well as any other option you'd like. +1. Click **Create** at the bottom of the page. +1. Click the name of your newly created VM Instance (optionally you can filter to find it). +1. Click **Set Windows password**. +1. Optional: Set a username or use default. +1. Click **Next**. +1. Copy and save the password as it won't be shown again. +1. Click **RDP** down arrow. +1. Click **Download the RDP file**. +1. Open the downloaded RDP file with the Windows remote desktop app (<https://docs.microsoft.com/en-us/windows-server/remote/remote-desktop-services/clients/remote-desktop-clients>). +1. Click **Continue** to accept the certificate. +1. Enter the password and click **Next**. + +You should now be remoted into a Windows machine with a command prompt. + +### Optional: Use GCP VM Instance as a runner + +- Register the runner with a project: `gitlab-runner.exe register`. +- Install the runner:`gitlab-runner.exe install`. +- Start the runner: `gitlab-runner.exe start`. + +For more information, see [Install GitLab Runner on Windows](https://docs.gitlab.com/runner/install/windows.html) +and [Registering Runners](https://docs.gitlab.com/runner/register/index.html). + +## Developer tips + +Here are a few tips on GCP and Windows. + +### GCP cost savings + +To minimise the cost of your GCP VM instance, stop it when you're not using it. +If you do, you'll need to redownload the RDP file from the console as the IP +address changes every time you stop and start it. + +### chocolatey + +Chocolatey is a package manager for Windows. You can search for packages on <https://chocolatey.org/>. + +- `choco install vim` + +### Visual Studio (install / usage for full GUI) + +You can install Visual Studio and run it within the Windows Remote Desktop app. + +Install it by running: `choco install visualstudio2019community` + +Start it by running: `"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\IDE\devenv.exe" .` + +### .NET 3 support + +You can install .NET version 3 support with the following `DISM` command: + +`DISM /Online /Enable-Feature /FeatureName:NetFx3 /All` + +### nix -> Windows cmd tips + +The first tip for using the Windows command shell is to open Powershell and use that instead. + +Start Powershell: `start powershell`. + +Powershell has aliases for all of the following commands so you don't have to learn the native commands: + +- `ls` ---> `dir` +- `rm` ---> `del` +- `rm -rf nonemptydir` ---> `rmdir /S nonemptydir` +- `/` ---> `\` (path separator) +- `cat` ---> `type` +- `mv` ---> `move` +- Redirection works the same (i.e. `>` and `2>&1`) +- `.\some.exe` to call a local executable +- curl is available +- `..` and `.` are available diff --git a/doc/downgrade_ee_to_ce/README.md b/doc/downgrade_ee_to_ce/README.md index 29f9baae5c4..cba21668816 100644 --- a/doc/downgrade_ee_to_ce/README.md +++ b/doc/downgrade_ee_to_ce/README.md @@ -48,15 +48,18 @@ to avoid getting this error, you need to remove all instances of the **Omnibus Installation** ```shell -sudo gitlab-rails runner "Service.where(type: ['JenkinsService', 'JenkinsDeprecatedService', 'GithubService']).delete_all" +sudo gitlab-rails runner "Service.where(type: ['JenkinsService', 'GithubService']).delete_all" ``` **Source Installation** ```shell -bundle exec rails runner "Service.where(type: ['JenkinsService', 'JenkinsDeprecatedService', 'GithubService']).delete_all" production +bundle exec rails runner "Service.where(type: ['JenkinsService', 'GithubService']).delete_all" production ``` +NOTE: **Note:** +If you are running `GitLab =< v13.0` you need to also remove `JenkinsDeprecatedService` records. + ### Variables environment scopes If you're using this feature and there are variables sharing the same diff --git a/doc/gitlab-basics/command-line-commands.md b/doc/gitlab-basics/command-line-commands.md index fed91046c32..b9426ec0849 100644 --- a/doc/gitlab-basics/command-line-commands.md +++ b/doc/gitlab-basics/command-line-commands.md @@ -2,7 +2,7 @@ type: howto, reference --- -# Command Line basic commands +# Edit files through the command line When [working with Git from the command line](start-using-git.md), you will need to use more than just the Git commands. There are several basic commands that you should diff --git a/doc/gitlab-basics/create-project.md b/doc/gitlab-basics/create-project.md index 1febe8337bc..152fb24699e 100644 --- a/doc/gitlab-basics/create-project.md +++ b/doc/gitlab-basics/create-project.md @@ -145,7 +145,7 @@ git push --set-upstream https://gitlab.example.com/namespace/nonexistent-project Once the push finishes successfully, a remote message will indicate the command to set the remote and the URL to the new project: -```text +```plaintext remote: remote: The private project namespace/nonexistent-project was created. remote: diff --git a/doc/gitlab-basics/create-your-ssh-keys.md b/doc/gitlab-basics/create-your-ssh-keys.md index 9b3431a5a42..248f39b842c 100644 --- a/doc/gitlab-basics/create-your-ssh-keys.md +++ b/doc/gitlab-basics/create-your-ssh-keys.md @@ -11,7 +11,7 @@ In order to use SSH, you will need to: ## Creating your SSH key pair -1. Go to your [command line](start-using-git.md#open-a-shell). +1. Go to your [command line](start-using-git.md#command-shell). 1. Follow the [instructions](../ssh/README.md#generating-a-new-ssh-key-pair) to generate your SSH key pair. diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 9a05c7fff14..9ebcf258ee9 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -1,81 +1,104 @@ --- type: howto, tutorial +description: "Introduction to using Git through the command line." +last_updated: 2020-04-22 --- # Start using Git on the command line -While GitLab has a powerful user interface, if you want to use Git itself, you will -have to do so from the command line. If you want to start using Git and GitLab together, -make sure that you have created and/or signed into an account on GitLab. +[Git](https://git-scm.com/) is an open-source distributed version control system designed to +handle everything from small to very large projects with speed and efficiency. GitLab is built +on top of Git. -## Open a shell +While GitLab has a powerful user interface from which you can do a great amount of Git operations +directly in the browser, you’ll eventually need to use Git through the command line for advanced +tasks. -Depending on your operating system, you will need to use a shell of your preference. -Here are some suggestions: +For example, if you need to fix complex merge conflicts, rebase branches, +merge manually, or undo and roll back commits, you'll need to use Git from +the command line and then push your changes to the remote server. -- [Terminal](https://blog.teamtreehouse.com/introduction-to-the-mac-os-x-command-line) on macOS -- [GitBash](https://msysgit.github.io) on Windows -- [Linux Terminal](https://www.howtogeek.com/140679/beginner-geek-how-to-start-using-the-linux-terminal/) on Linux +This guide will help you get started with Git through the command line and can be your reference +for Git commands in the future. If you're only looking for a quick reference of Git commands, you +can download GitLab's [Git Cheat Sheet](https://about.gitlab.com/images/press/git-cheat-sheet.pdf). -## Check if Git has already been installed +TIP: **Tip:** +To help you visualize what you're doing locally, there are +[Git GUI apps](https://git-scm.com/download/gui/) you can install. -Git is usually preinstalled on Mac and Linux, so run the following command: +## Requirements -```shell -git --version -``` +You don't need a GitLab account to use Git locally, but for the purpose of this guide we +recommend registering and signing into your account before starting. Some commands need a +connection between the files in your computer and their version on a remote server. -You should receive a message that tells you which Git version you have on your computer. -If you don’t receive a "Git version" message, it means that you need to -[download Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). +You'll also need to open a [command shell](#command-shell) and have +[Git installed](#install-git) in your computer. -After you are finished installing Git, open a new shell and type `git --version` again -to verify that it was correctly installed. +### Command shell -## Add your Git username and set your email +To execute Git commands in your computer, you'll need to open a command shell (also known as command +prompt, terminal, and command line) of your preference. Here are some suggestions: -It is important to configure your Git username and email address, since every Git -commit will use this information to identify you as the author. +- For macOS users: + - Built-in: [Terminal](https://blog.teamtreehouse.com/introduction-to-the-mac-os-x-command-line). Press <kbd>⌘ command</kbd> + <kbd>space</kbd> and type "terminal" to find it. + - [iTerm2](https://www.iterm2.com/), which you can integrate with [zsh](https://git-scm.com/book/id/v2/Appendix-A%3A-Git-in-Other-Environments-Git-in-Zsh) and [oh my zsh](https://ohmyz.sh/) for color highlighting, among other handy features for Git users. +- For Windows users: + - Built-in: **cmd**. Click the search icon on the bottom navbar on Windows and type "cmd" to find it. + - [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-windows-powershell?view=powershell-7): a Windows "powered up" shell, from which you can execute a greater number of commands. + - Git Bash: it comes built into [Git for Windows](https://gitforwindows.org/). +- For Linux users: + - Built-in: [Linux Terminal](https://www.howtogeek.com/140679/beginner-geek-how-to-start-using-the-linux-terminal/). -In your shell, type the following command to add your username: +### Install Git + +Open a command shell and run the following command to check if Git is already installed in your +computer: ```shell -git config --global user.name "YOUR_USERNAME" +git --version ``` -Then verify that you have the correct username: +If you have Git installed, the output will be: ```shell -git config --global user.name +git version X.Y.Z ``` -To set your email address, type the following command: +If your computer doesn't recognize `git` as a command, you'll need to [install Git](../topics/git/how_to_install_git/index.md). +After that, run `git --version` again to verify whether it was correctly installed. -```shell -git config --global user.email "your_email_address@example.com" -``` +## Configure Git + +To start using Git from your computer, you'll need to enter your credentials (user name and email) +to identify you as the author of your work. The user name and email should match the ones you're +using on GitLab. -To verify that you entered your email correctly, type: +In your shell, add your user name: ```shell -git config --global user.email +git config --global user.name "your_username" ``` -You'll need to do this only once, since you are using the `--global` option. It -tells Git to always use this information for anything you do on that system. If -you want to override this with a different username or email address for specific -projects or repositories, you can run the command without the `--global` option -when you’re in that project, and that will default to `--local`. You can read more -on how Git manages configurations in the [Git Config](https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration) documentation. +And your email address: -## Check your information +```shell +git config --global user.email "your_email_address@example.com" +``` -To view the information that you entered, along with other global options, type: +To check the configuration, run: ```shell git config --global --list ``` +The `--global` option tells Git to always use this information for anything you do on your system. +If you omit `--global` or use `--local`, the configuration will be applied only to the current +repository. + +You can read more on how Git manages configurations in the +[Git Config](https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration) documentation. + ## Basic Git commands Start using Git via the command line with the most basic commands as described below. @@ -103,11 +126,11 @@ To start working locally on an existing remote repository, clone it with the com files to your local computer, automatically preserving the Git connection with the remote repository. -You can either clone it via HTTPS or [SSH](../ssh/README.md). If you chose to clone -it via HTTPS, you'll have to enter your credentials every time you pull and push. +You can either clone it via [HTTPS](#clone-via-https) or [SSH](#clone-via-ssh). If you chose to +clone it via HTTPS, you'll have to enter your credentials every time you pull and push. You can read more about credential storage in the [Git Credentials documentation](https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage). -With SSH, you enter your credentials only once. +With [SSH](../ssh/README.md), you enter your credentials only once. You can find both paths (HTTPS and SSH) by navigating to your project's landing page and clicking **Clone**. GitLab will prompt you with both paths, from which you can copy @@ -119,24 +142,38 @@ As an example, consider this repository path: - SSH: `git@gitlab.com:gitlab-org/gitlab.git` To get started, open a terminal window in the directory you wish to clone the -repository files into, and run one of the following commands. +repository files into, and run one of the `git clone` commands as described below. + +Both commands will download a copy of the files in a folder named after the project's +name. You can then navigate to the new directory and start working on it locally. + +#### Clone via HTTPS -Clone via HTTPS: +To clone `https://gitlab.com/gitlab-org/gitlab.git` via HTTPS: ```shell git clone https://gitlab.com/gitlab-org/gitlab.git ``` -Clone via SSH: +You'll have to add your password every time you clone through HTTPS. If you have 2FA enabled +for your account, you'll have to use a [Personal Access Token](../user/profile/personal_access_tokens.md) +with **read_repository** or **write_repository** permissions instead of your account's password. + +If you don't have 2FA enabled, use your account's password. + +TIP: **Troubleshooting:** +On Windows, if you entered incorrect passwords multiple times and GitLab is responding `Access denied`, +you may have to add your namespace (user name or group name) to clone through HTTPS: +`git clone https://namespace@gitlab.com/gitlab-org/gitlab.git`. + +#### Clone via SSH + +To clone `git@gitlab.com:gitlab-org/gitlab.git` via SSH: ```shell git clone git@gitlab.com:gitlab-org/gitlab.git ``` -Both commands will download a copy of the files in a folder named after the project's -name. You can then navigate to the directory and start working -on it locally. - ### Switch to the master branch You are always in a branch when working with Git. The main branch is the master diff --git a/doc/gitlab-geo/ha.md b/doc/gitlab-geo/ha.md index 23ed11eaf09..0be70791d45 100644 --- a/doc/gitlab-geo/ha.md +++ b/doc/gitlab-geo/ha.md @@ -1,5 +1,5 @@ --- -redirect_to: '../administration/geo/replication/high_availability.md' +redirect_to: '../administration/geo/replication/multiple_servers.md' --- -This document was moved to [another location](../administration/geo/replication/high_availability.md). +This document was moved to [another location](../administration/geo/replication/multiple_servers.md). diff --git a/doc/install/README.md b/doc/install/README.md index 83910b0a281..f1ef368685e 100644 --- a/doc/install/README.md +++ b/doc/install/README.md @@ -14,15 +14,15 @@ and cost of hosting. There are many ways you can install GitLab depending on your platform: 1. **Omnibus GitLab**: The official deb/rpm packages that contain a bundle of GitLab - and the various components it depends on like PostgreSQL, Redis, Sidekiq, etc. + and the various components it depends on, like PostgreSQL, Redis, Sidekiq, etc. 1. **GitLab Helm chart**: The cloud native Helm chart for installing GitLab and all its components on Kubernetes. 1. **Docker**: The Omnibus GitLab packages dockerized. 1. **Source**: Install GitLab and all its components from scratch. TIP: **If in doubt, choose Omnibus:** -The Omnibus GitLab packages are mature, scalable, support -[high availability](../administration/availability/index.md) and are used +The Omnibus GitLab packages are mature, +[scalable](../administration/reference_architectures/index.md) and are used today on GitLab.com. The Helm charts are recommended for those who are familiar with Kubernetes. @@ -36,7 +36,7 @@ The Omnibus GitLab package uses our official deb/rpm repositories. This is recommended for most users. If you need additional flexibility and resilience, we recommend deploying -GitLab as described in our [High Availability documentation](../administration/availability/index.md). +GitLab as described in our [reference architecture documentation](../administration/reference_architectures/index.md). [**> Install GitLab using the Omnibus GitLab package.**](https://about.gitlab.com/install/) @@ -66,8 +66,8 @@ GitLab maintains a set of official Docker images based on the Omnibus GitLab pac ## Installing GitLab from source -If the GitLab Omnibus package is not available in your distribution, you can -install GitLab from source: Useful for unsupported systems like *BSD. For an +If the Omnibus GitLab package is not available in your distribution, you can +install GitLab from source: Useful for unsupported systems like \*BSD. For an overview of the directory structure, read the [structure documentation](structure.md). [**> Install GitLab from source.**](installation.md) diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 48de5e274b0..41f8d7babac 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -2,9 +2,9 @@ type: howto --- -# Installing GitLab HA on Amazon Web Services (AWS) +# Installing GitLab on Amazon Web Services (AWS) -This page offers a walkthrough of a common HA (Highly Available) configuration +This page offers a walkthrough of a common configuration for GitLab on AWS. You should customize it to accommodate your needs. NOTE: **Note** @@ -13,11 +13,10 @@ For organizations with 300 users or less, the recommended AWS installation metho ## Introduction GitLab on AWS can leverage many of the services that are already -configurable with GitLab High Availability (HA). These services offer a great deal of -flexibility and can be adapted to the needs of most companies, while enabling the -automation of both vertical and horizontal scaling. +configurable. These services offer a great deal of +flexibility and can be adapted to the needs of most companies. -In this guide, we'll go through a basic HA setup where we'll start by +In this guide, we'll go through a multi-node setup where we'll start by configuring our Virtual Private Cloud and subnets to later integrate services such as RDS for our database server and ElastiCache as a Redis cluster to finally manage them within an auto scaling group with custom @@ -54,26 +53,60 @@ Here's a list of the AWS services we will use, with links to pricing information [Amazon S3 pricing](https://aws.amazon.com/s3/pricing/). - **ELB**: A Classic Load Balancer will be used to route requests to the GitLab instances. See the [Amazon ELB pricing](https://aws.amazon.com/elasticloadbalancing/pricing/). -- **RDS**: An Amazon Relational Database Service using PostgreSQL will be used - to provide a High Availability database configuration. See the +- **RDS**: An Amazon Relational Database Service using PostgreSQL will be used. See the [Amazon RDS pricing](https://aws.amazon.com/rds/postgresql/pricing/). - **ElastiCache**: An in-memory cache environment will be used to provide a - High Availability Redis configuration. See the + Redis configuration. See the [Amazon ElastiCache pricing](https://aws.amazon.com/elasticache/pricing/). NOTE: **Note:** Please note that while we will be using EBS for storage, we do not recommend using EFS as it may negatively impact GitLab's performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. -## Creating an IAM EC2 instance role and profile +## Create an IAM EC2 instance role and profile + +As we'll be using [Amazon S3 object storage](#amazon-s3-object-storage), our EC2 instances need to have read, write, and list permissions for our S3 buckets. To avoid embedding AWS keys in our GitLab config, we'll make use of an [IAM Role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) to allow our GitLab instance with this access. We'll need to create an IAM policy to attach to our IAM role: + +### Create an IAM Policy + +1. Navigate to the IAM dashboard and click on **Policies** in the left menu. +1. Click **Create policy**, select the `JSON` tab, and add a policy. We want to [follow security best practices and grant _least privilege_](https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#grant-least-privilege), giving our role only the permissions needed to perform the required actions. + 1. Assuming you prefix the S3 bucket names with `gl-` as shown in the diagram, add the following policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:CompleteMultipartUpload", + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::gl-*/*" + ] + } + ] +} +``` + +1. Click **Review policy**, give your policy a name (we'll use `gl-s3-policy`), and click **Create policy**. -To minimize the permissions of the user, we'll create a new [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html) -role with limited access: +### Create an IAM Role -1. Navigate to the IAM dashboard <https://console.aws.amazon.com/iam/home> and +1. Still on the IAM dashboard, click on **Roles** in the left menu, and click **Create role**. 1. Create a new role by selecting **AWS service > EC2**, then click **Next: Permissions**. -1. Choose **AmazonEC2FullAccess** and **AmazonS3FullAccess**, then click **Next: Review**. -1. Give the role the name `GitLabAdmin` and click **Create role**. +1. In the policy filter, search for the `gl-s3-policy` we created above, select it, and click **Tags**. +1. Add tags if needed and click **Review**. +1. Give the role a name (we'll use `GitLabS3Access`) and click **Create Role**. + +We'll use this role when we [create a launch configuration](#create-a-launch-configuration) later on. ## Configuring the network @@ -94,6 +127,8 @@ We'll now create a VPC, a virtual networking environment that you'll control:  +1. Select the VPC, click **Actions**, click **Edit DNS resolution**, and enable DNS resolution. Hit **Save** when done. + ### Subnets Now, let's create some subnets in different Availability Zones. Make sure @@ -106,7 +141,7 @@ RDS instances as well: 1. Select **Subnets** from the left menu. 1. Click **Create subnet**. Give it a descriptive name tag based on the IP, - for example `gitlab-public-10.0.0.0`, select the VPC we created previously, + for example `gitlab-public-10.0.0.0`, select the VPC we created previously, select an availability zone (we'll use `us-west-2a`), and at the IPv4 CIDR block let's give it a 24 subnet `10.0.0.0/24`:  @@ -120,18 +155,8 @@ RDS instances as well: | `gitlab-public-10.0.2.0` | public | `us-west-2b` | `10.0.2.0/24` | | `gitlab-private-10.0.3.0` | private | `us-west-2b` | `10.0.3.0/24` | -### Create NAT Gateways - -Instances deployed in our private subnets need to connect to the internet for updates, but should not be reachable from the public internet. To achieve this, we'll make use of [NAT Gateways](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway.html) deployed in each of our public subnets: - -1. Navigate to the VPC dashboard and click on **NAT Gateways** in the left menu bar. -1. Click **Create NAT Gateway** and complete the following: - 1. **Subnet**: Select `gitlab-public-10.0.0.0` from the dropdown. - 1. **Elastic IP Allocation ID**: Enter an existing Elastic IP or click **Allocate Elastic IP address** to allocate a new IP to your NAT gateway. - 1. Add tags if needed. - 1. Click **Create NAT Gateway**. - -Create a second NAT gateway but this time place it in the second public subnet, `gitlab-public-10.0.2.0`. +1. Once all the subnets are created, enable **Auto-assign IPv4** for the two public subnets: + 1. Select each public subnet in turn, click **Actions**, and click **Modify auto-assign IP settings**. Enable the option and save. ### Internet Gateway @@ -148,6 +173,19 @@ create a new one: 1. Choose `gitlab-vpc` from the list and hit **Attach**. +### Create NAT Gateways + +Instances deployed in our private subnets need to connect to the internet for updates, but should not be reachable from the public internet. To achieve this, we'll make use of [NAT Gateways](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway.html) deployed in each of our public subnets: + +1. Navigate to the VPC dashboard and click on **NAT Gateways** in the left menu bar. +1. Click **Create NAT Gateway** and complete the following: + 1. **Subnet**: Select `gitlab-public-10.0.0.0` from the dropdown. + 1. **Elastic IP Allocation ID**: Enter an existing Elastic IP or click **Allocate Elastic IP address** to allocate a new IP to your NAT gateway. + 1. Add tags if needed. + 1. Click **Create NAT Gateway**. + +Create a second NAT gateway but this time place it in the second public subnet, `gitlab-public-10.0.2.0`. + ### Route Tables #### Public Route Table @@ -179,13 +217,13 @@ Next, we must associate the **public** subnets to the route table: We also need to create two private route tables so that instances in each private subnet can reach the internet via the NAT gateway in the corresponding public subnet in the same availability zone. -1. Follow the same steps as above to create two private route tables. Name them `gitlab-public-a` and `gitlab-public-b` respectively. +1. Follow the same steps as above to create two private route tables. Name them `gitlab-private-a` and `gitlab-private-b` respectively. 1. Next, add a new route to each of the private route tables where the destination is `0.0.0.0/0` and the target is one of the NAT gateways we created earlier. - 1. Add the NAT gateway we created in `gitlab-public-10.0.0.0` as the target for the new route in the `gitlab-public-a` route table. - 1. Similarly, add the NAT gateway in `gitlab-public-10.0.2.0` as the target for the new route in the `gitlab-public-b`. + 1. Add the NAT gateway we created in `gitlab-public-10.0.0.0` as the target for the new route in the `gitlab-private-a` route table. + 1. Similarly, add the NAT gateway in `gitlab-public-10.0.2.0` as the target for the new route in the `gitlab-private-b`. 1. Lastly, associate each private subnet with a private route table. - 1. Associate `gitlab-private-10.0.1.0` with `gitlab-public-a`. - 1. Associate `gitlab-private-10.0.3.0` with `gitlab-public-b`. + 1. Associate `gitlab-private-10.0.1.0` with `gitlab-private-a`. + 1. Associate `gitlab-private-10.0.3.0` with `gitlab-private-b`. ## Load Balancer @@ -198,7 +236,7 @@ On the EC2 dashboard, look for Load Balancer in the left navigation bar: 1. In the **Select Subnets** section, select both public subnets from the list. 1. Click **Assign Security Groups** and select **Create a new security group**, give it a name (we'll use `gitlab-loadbalancer-sec-group`) and description, and allow both HTTP and HTTPS traffic - from anywhere (`0.0.0.0/0, ::/0`). + from anywhere (`0.0.0.0/0, ::/0`). Also allow SSH traffic from a single IP address or an IP address range in CIDR notation. 1. Click **Configure Security Settings** and select an SSL/TLS certificate from ACM or upload a certificate to IAM. 1. Click **Configure Health Check** and set up a health check for your EC2 instances. 1. For **Ping Protocol**, select HTTP. @@ -232,19 +270,9 @@ On the Route 53 dashboard, click **Hosted zones** in the left navigation bar: ## PostgreSQL with RDS For our database server we will use Amazon RDS which offers Multi AZ -for redundancy. Let's start by creating a subnet group and then we'll +for redundancy. First we'll create a security group and subnet group, then we'll create the actual RDS instance. -### RDS Subnet Group - -1. Navigate to the RDS dashboard and select **Subnet Groups** from the left menu. -1. Click on **Create DB Subnet Group**. -1. Under **Subnet group details**, enter a name (we'll use `gitlab-rds-group`), a description, and choose the `gitlab-vpc` from the VPC dropdown. -1. Under **Add subnets**, click **Add all the subnets related to this VPC** and remove the public ones, we only want the **private subnets**. In the end, you should see `10.0.1.0/24` and `10.0.3.0/24` (as we defined them in the [subnets section](#subnets)). -1. Click **Create** when ready. - -  - ### RDS Security Group We need a security group for our database that will allow inbound traffic from the instances we'll deploy in our `gitlab-loadbalancer-sec-group` later on: @@ -255,21 +283,33 @@ We need a security group for our database that will allow inbound traffic from t 1. In the **Inbound rules** section, click **Add rule** and add a **PostgreSQL** rule, and set the "Custom" source as the `gitlab-loadbalancer-sec-group` we created earlier. The default PostgreSQL port is `5432`, which we'll also use when creating our database below. 1. When done, click **Create security group**. +### RDS Subnet Group + +1. Navigate to the RDS dashboard and select **Subnet Groups** from the left menu. +1. Click on **Create DB Subnet Group**. +1. Under **Subnet group details**, enter a name (we'll use `gitlab-rds-group`), a description, and choose the `gitlab-vpc` from the VPC dropdown. +1. Under **Add subnets**, click **Add all the subnets related to this VPC** and remove the public ones, we only want the **private subnets**. In the end, you should see `10.0.1.0/24` and `10.0.3.0/24` (as we defined them in the [subnets section](#subnets)). +1. Click **Create** when ready. + +  + ### Create the database +DANGER: **Danger:** Avoid using burstable instances (t class instances) for the database as this could lead to performance issues due to CPU credits running out during sustained periods of high load. + Now, it's time to create the database: -1. Select **Databases** from the left menu and click **Create database**. +1. Navigate to the RDS dashboard, select **Databases** from the left menu, and click **Create database**. 1. Select **Standard Create** for the database creation method. 1. Select **PostgreSQL** as the database engine and select **PostgreSQL 10.9-R1** from the version dropdown menu (check the [database requirements](../../install/requirements.md#postgresql-requirements) to see if there are any updates on this for your chosen version of GitLab). 1. Since this is a production server, let's choose **Production** from the **Templates** section. 1. Under **Settings**, set a DB instance identifier, a master username, and a master password. We'll use `gitlab-db-ha`, `gitlab`, and a very secure password respectively. Make a note of these as we'll need them later. 1. For the DB instance size, select **Standard classes** and select an instance size that meets your requirements from the dropdown menu. We'll use a `db.m4.large` instance. 1. Under **Storage**, configure the following: - 1. Select **Provisioned IOPS (SSD)** from the storage type dropdown menu. Provisioned IOPS (SSD) storage is best suited for HA (though you can choose General Purpose (SSD) to reduce the costs). Read more about it at [Storage for Amazon RDS](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html). + 1. Select **Provisioned IOPS (SSD)** from the storage type dropdown menu. Provisioned IOPS (SSD) storage is best suited for this use (though you can choose General Purpose (SSD) to reduce the costs). Read more about it at [Storage for Amazon RDS](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html). 1. Allocate storage and set provisioned IOPS. We'll use the minimum values, `100` and `1000`, respectively. 1. Enable storage autoscaling (optional) and set a maximum storage threshold. -1. Under **Availability & durability**, select **Create a standby instance** to have a standby RDS instance provisioned in a different Availability Zone. Read more at [High Availability (Multi-AZ)](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html). +1. Under **Availability & durability**, select **Create a standby instance** to have a standby RDS instance provisioned in a different [Availability Zone](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html). 1. Under **Connectivity**, configure the following: 1. Select the VPC we created earlier (`gitlab-vpc`) from the **Virtual Private Cloud (VPC)** dropdown menu. 1. Expand the **Additional connectivity configuration** section and select the subnet group (`gitlab-rds-group`) we created earlier. @@ -291,15 +331,6 @@ Now that the database is created, let's move on to setting up Redis with ElastiC ElastiCache is an in-memory hosted caching solution. Redis maintains its own persistence and is used for certain types of the GitLab application. -### Redis Subnet Group - -1. Navigate to the ElastiCache dashboard from your AWS console. -1. Go to **Subnet Groups** in the left menu, and create a new subnet group. - Make sure to select our VPC and its [private subnets](#subnets). Click - **Create** when ready. - -  - ### Create a Redis Security Group 1. Navigate to the EC2 dashboard. @@ -309,6 +340,15 @@ persistence and is used for certain types of the GitLab application. 1. In the **Inbound rules** section, click **Add rule** and add a **Custom TCP** rule, set port `6379`, and set the "Custom" source as the `gitlab-loadbalancer-sec-group` we created earlier. 1. When done, click **Create security group**. +### Redis Subnet Group + +1. Navigate to the ElastiCache dashboard from your AWS console. +1. Go to **Subnet Groups** in the left menu, and create a new subnet group (we'll name ours `gitlab-redis-group`). + Make sure to select our VPC and its [private subnets](#subnets). Click + **Create** when ready. + +  + ### Create the Redis Cluster 1. Navigate back to the ElastiCache dashboard. @@ -392,7 +432,7 @@ From the EC2 dashboard: 1. Select an instance type based on your workload. Consult the [hardware requirements](../../install/requirements.md#hardware-requirements) to choose one that fits your needs (at least `c5.xlarge`, which is sufficient to accommodate 100 users). 1. Click **Configure Instance Details**: 1. In the **Network** dropdown, select `gitlab-vpc`, the VPC we created earlier. - 1. In the **Subnet** dropdown, `select gitlab-private-10.0.1.0` from the list of subnets we created earlier. + 1. In the **Subnet** dropdown, select `gitlab-private-10.0.1.0` from the list of subnets we created earlier. 1. Double check that **Auto-assign Public IP** is set to `Use subnet setting (Disable)`. 1. Click **Add Storage**. 1. The root volume is 8GiB by default and should be enough given that we won’t store any data there. @@ -405,6 +445,22 @@ From the EC2 dashboard: Connect to your GitLab instance via **Bastion Host A** using [SSH Agent Forwarding](#use-ssh-agent-forwarding). Once connected, add the following custom configuration: +#### Disable Let's Encrypt + +Since we're adding our SSL certificate at the load balancer, we do not need GitLab's built-in support for Let's Encrypt. Let's Encrypt [is enabled by default](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration) when using an `https` domain since GitLab 10.7, so we need to explicitly disable it: + +1. Open `/etc/gitlab/gitlab.rb` and disable it: + + ```ruby + letsencrypt['enable'] = false + ``` + +1. Save the file and reconfigure for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + #### Install the `pg_trgm` extension for PostgreSQL From your GitLab instance, connect to the RDS instance to verify access and to install the required `pg_trgm` extension. @@ -477,7 +533,7 @@ gitlab=# \q #### Set up Gitaly -CAUTION: **Caution:** In this architecture, having a single Gitaly server creates a single point of failure. This limitation will be removed once [Gitaly HA](https://gitlab.com/groups/gitlab-org/-/epics/842) is released. +CAUTION: **Caution:** In this architecture, having a single Gitaly server creates a single point of failure. This limitation will be removed once [Gitaly Cluster](https://gitlab.com/groups/gitlab-org/-/epics/1489) is released. Gitaly is a service that provides high-level RPC access to Git repositories. It should be enabled and configured on a separate EC2 instance in one of the @@ -499,6 +555,7 @@ Let's create an EC2 instance where we'll install Gitaly: 1. Click on **Configure Security Group** and let's **Create a new security group**. 1. Give your security group a name and description. We'll use `gitlab-gitaly-sec-group` for both. 1. Create a **Custom TCP** rule and add port `8075` to the **Port Range**. For the **Source**, select the `gitlab-loadbalancer-sec-group`. + 1. Also add an inbound rule for SSH from the `bastion-sec-group` so that we can connect using [SSH Agent Forwarding](#use-ssh-agent-forwarding) from the Bastion hosts. 1. Click **Review and launch** followed by **Launch** if you're happy with your settings. 1. Finally, acknowledge that you have access to the selected private key file or create a new one. Click **Launch Instances**. @@ -512,48 +569,51 @@ As we are terminating SSL at our [load balancer](#load-balancer), follow the ste Remember to run `sudo gitlab-ctl reconfigure` after saving the changes to the `gitlab.rb` file. -#### Disable Let's Encrypt +#### Fast lookup of authorized SSH keys -Since we're adding our SSL certificate at the load balancer, we do not need GitLab's built-in support for Let's Encrypt. Let's Encrypt [is enabled by default](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration) when using an `https` domain since GitLab 10.7, so we need to explicitly disable it: +The public SSH keys for users allowed to access GitLab are stored in `/var/opt/gitlab/.ssh/authorized_keys`. Typically we'd use shared storage so that all the instances are able to access this file when a user performs a Git action over SSH. Since we do not have shared storage in our setup, we'll update our configuration to authorize SSH users via indexed lookup in the GitLab database. -1. Open `/etc/gitlab/gitlab.rb` and disable it: +Follow the instructions at [Setting up fast lookup via GitLab Shell](../../administration/operations/fast_ssh_key_lookup.md#setting-up-fast-lookup-via-gitlab-shell) to switch from using the `authorized_keys` file to the database. - ```ruby - letsencrypt['enable'] = false - ``` +If you do not configure fast lookup, Git actions over SSH will result in the following error: -1. Save the file and reconfigure for the changes to take effect: +```shell +Permission denied (publickey). +fatal: Could not read from remote repository. - ```shell - sudo gitlab-ctl reconfigure - ``` +Please make sure you have the correct access rights +and the repository exists. +``` #### Configure host keys -Ordinarily we would manually copy the contents (primary and public keys) of `/etc/ssh/` on the primary application server to `/etc/ssh` on all secondary servers. This prevents false man-in-the-middle-attack alerts when accessing servers in your High Availability cluster behind a load balancer. +Ordinarily we would manually copy the contents (primary and public keys) of `/etc/ssh/` on the primary application server to `/etc/ssh` on all secondary servers. This prevents false man-in-the-middle-attack alerts when accessing servers in your cluster behind a load balancer. We'll automate this by creating static host keys as part of our custom AMI. As these host keys are also rotated every time an EC2 instance boots up, "hard coding" them into our custom AMI serves as a handy workaround. On your GitLab instance run the following: ```shell -mkdir /etc/ssh_static -cp -R /etc/ssh/* /etc/ssh_static +sudo mkdir /etc/ssh_static +sudo cp -R /etc/ssh/* /etc/ssh_static ``` In `/etc/ssh/sshd_config` update the following: ```bash - # HostKeys for protocol version 2 - HostKey /etc/ssh_static/ssh_host_rsa_key - HostKey /etc/ssh_static/ssh_host_dsa_key - HostKey /etc/ssh_static/ssh_host_ecdsa_key - HosstKey /etc/ssh_static/ssh_host_ed25519_key +# HostKeys for protocol version 2 +HostKey /etc/ssh_static/ssh_host_rsa_key +HostKey /etc/ssh_static/ssh_host_dsa_key +HostKey /etc/ssh_static/ssh_host_ecdsa_key +HostKey /etc/ssh_static/ssh_host_ed25519_key ``` #### Amazon S3 object storage -Since we're not using NFS for shared storage, we will use [Amazon S3](https://aws.amazon.com/s3/) buckets to store backups, artifacts, LFS objects, uploads, merge request diffs, container registry images, and more. Our [documentation includes configuration instructions](../../administration/object_storage.md) for each of these, and other information about using object storage with GitLab. +Since we're not using NFS for shared storage, we will use [Amazon S3](https://aws.amazon.com/s3/) buckets to store backups, artifacts, LFS objects, uploads, merge request diffs, container registry images, and more. Our documentation includes [instructions on how to configure object storage](../../administration/object_storage.md) for each of these data types, and other information about using object storage with GitLab. + +NOTE: **Note:** +Since we are using the [AWS IAM profile](#create-an-iam-role) we created earlier, be sure to omit the AWS access key and secret access key/value pairs when configuring object storage. Instead, use `'use_iam_profile' => true` in your configuration as shown in the object storage documentation linked above. Remember to run `sudo gitlab-ctl reconfigure` after saving the changes to the `gitlab.rb` file. @@ -589,7 +649,7 @@ From the EC2 dashboard: 1. Select an instance type best suited for your needs (at least a `c5.xlarge`) and click **Configure details**. 1. Enter a name for your launch configuration (we'll use `gitlab-ha-launch-config`). 1. **Do not** check **Request Spot Instance**. -1. From the **IAM Role** dropdown, pick the `GitLabAdmin` instance role we [created earlier](#creating-an-iam-ec2-instance-role-and-profile). +1. From the **IAM Role** dropdown, pick the `GitLabAdmin` instance role we [created earlier](#create-an-iam-ec2-instance-role-and-profile). 1. Leave the rest as defaults and click **Add Storage**. 1. The root volume is 8GiB by default and should be enough given that we won’t store any data there. Click **Configure Security Group**. 1. Check **Select and existing security group** and select the `gitlab-loadbalancer-sec-group` we created earlier. @@ -604,7 +664,7 @@ From the EC2 dashboard: 1. Select the `gitlab-vpc` from the **Network** dropdown. 1. Add both the private [subnets we created earlier](#subnets). 1. Expand the **Advanced Details** section and check the **Receive traffic from one or more load balancers** option. -1. From the **Classic Load Balancers** dropdown, Select the load balancer we created earlier. +1. From the **Classic Load Balancers** dropdown, select the load balancer we created earlier. 1. For **Health Check Type**, select **ELB**. 1. We'll leave our **Health Check Grace Period** as the default `300` seconds. Click **Configure scaling policies**. 1. Check **Use scaling policies to adjust the capacity of this group**. @@ -635,7 +695,7 @@ GitLab provides its own integrated monitoring solution based on Prometheus. For more information on how to set it up, visit the [GitLab Prometheus documentation](../../administration/monitoring/prometheus/index.md) -GitLab also has various [health check endpoints](../..//user/admin_area/monitoring/health_check.md) +GitLab also has various [health check endpoints](../../user/admin_area/monitoring/health_check.md) that you can ping and get reports. ## GitLab Runners @@ -648,8 +708,8 @@ Read more on configuring an ## Backup and restore -GitLab provides [a tool to backup](../../raketasks/backup_restore.md#creating-a-backup-of-the-gitlab-system) -and restore its Git data, database, attachments, LFS objects, etc. +GitLab provides [a tool to back up](../../raketasks/backup_restore.md#back-up-gitlab) +and restore its Git data, database, attachments, LFS objects, and so on. Some important things to know: @@ -675,7 +735,7 @@ For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. ### Restoring GitLab from a backup -To restore GitLab, first review the [restore documentation](../../raketasks/backup_restore.md#restore), +To restore GitLab, first review the [restore documentation](../../raketasks/backup_restore.md#restore-gitlab), and primarily the restore prerequisites. Then, follow the steps under the [Omnibus installations section](../../raketasks/backup_restore.md#restore-for-omnibus-gitlab-installations). @@ -708,7 +768,7 @@ After a few minutes, the new version should be up and running. In this guide, we went mostly through scaling and some redundancy options, your mileage may vary. -Keep in mind that all Highly Available solutions come with a trade-off between +Keep in mind that all solutions come with a trade-off between cost/complexity and uptime. The more uptime you want, the more complex the solution. And the more complex the solution, the more work is involved in setting up and maintaining it. @@ -717,8 +777,8 @@ Have a read through these other resources and feel free to [open an issue](https://gitlab.com/gitlab-org/gitlab/issues/new) to request additional material: -- [Scaling GitLab](../../administration/scaling/index.md): - GitLab supports several different types of clustering and high-availability. +- [Scaling GitLab](../../administration/reference_architectures/index.md): + GitLab supports several different types of clustering. - [Geo replication](../../administration/geo/replication/index.md): Geo is the solution for widely distributed development teams. - [Omnibus GitLab](https://docs.gitlab.com/omnibus/) - Everything you need to know diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index b782f1c82cd..fbc81da20d4 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -21,7 +21,7 @@ First, you'll need an account on Azure. There are three ways to do this: services, exploring Microsoft's cloud for free. Even after the first 30 days, you never have to pay anything unless you decide to transition to paid services with a Pay-As-You-Go Azure subscription. This is a great way to try out Azure and cloud computing, and you can - [read more in their comprehensive FAQ][Azure-Free-Account-FAQ]. + [read more in their comprehensive FAQ](https://azure.microsoft.com/en-us/free/free-account-faq/). - If you have an MSDN subscription, you can activate your Azure subscriber benefits. Your MSDN subscription gives you recurring Azure credits every month, so why not put those credits to use and try out GitLab right now? @@ -73,7 +73,7 @@ The first items we need to configure are the basic settings of the underlying vi _(read the [SSH documentation](../../ssh/README.md) to learn more about how to set up SSH public keys)_ 1. If you chose **Password** - enter the password you wish to use _(this is the password that you - will use later in this tutorial to [SSH] into the VM, so make sure it's a strong password/passphrase)_ + will use later in this tutorial to [SSH](https://en.wikipedia.org/wiki/Secure_Shell) into the VM, so make sure it's a strong password/passphrase)_ 1. Choose the appropriate `Subscription` tier for your Azure account 1. Choose an existing `Resource Group` or create a new one - e.g. **"GitLab-CE-Azure"** @@ -177,7 +177,7 @@ Click **"Save"** for the changes to take effect. domain registrar which points to the public IP address of your Azure VM. If you do this, you'll need to make sure your VM is configured to use a _static_ public IP address (i.e. not a _dynamic_ one) or you will have to reconfigure the DNS `A` record each time Azure reassigns your VM a new public IP -address. Read [IP address types and allocation methods in Azure][Azure-IP-Address-Types] to learn more. +address. Read [IP address types and allocation methods in Azure](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-ip-addresses-overview-arm) to learn more. ## Let's open some ports @@ -216,7 +216,7 @@ ports to enable public internet access to two services in particular: public access to the instance of GitLab running on our VM. 1. **SSH** (port 22) - opening port 22 will enable our VM to respond to SSH connection requests, allowing public access (with authentication) to remote terminal sessions - _(you'll see why we need [SSH] access to our VM [later on in this tutorial](#maintaining-your-gitlab-instance))_ + _(you'll see why we need [SSH](https://en.wikipedia.org/wiki/Secure_Shell) access to our VM [later on in this tutorial](#maintaining-your-gitlab-instance))_ ### Open HTTP on Port 80 @@ -233,7 +233,7 @@ connections: ### Open SSH on Port 22 Repeat the above process, adding a second Inbound security rule to open port 22, enabling our VM to -accept [SSH] connections: +accept [SSH](https://en.wikipedia.org/wiki/Secure_Shell) connections:  @@ -327,7 +327,7 @@ process will still be the same. To perform an update, we need to connect directly to our Azure VM instance and run some commands from the terminal. Our Azure VM is actually a server running Linux (Ubuntu), so we'll need to -connect to it using SSH ([Secure Shell][SSH]). +connect to it using SSH ([Secure Shell](https://en.wikipedia.org/wiki/Secure_Shell)). If you're running Windows, you'll need to connect using [PuTTY](https://www.putty.org) or an equivalent Windows SSH client. If you're running Linux or macOS, then you already have an SSH client installed. @@ -341,7 +341,7 @@ If you're running Linux or macOS, then you already have an SSH client installed. #### SSH from the command-line -If you're running [SSH] from the command-line (terminal), then type in the following command to +If you're running [SSH](https://en.wikipedia.org/wiki/Secure_Shell) from the command-line (terminal), then type in the following command to connect to your VM, substituting `username` and `your-azure-domain-name.com` for the correct values. Again, remember that your Azure VM domain name will be the one you @@ -356,8 +356,8 @@ Provide your password at the prompt to authenticate. #### SSH from Windows (PuTTY) -If you're using [PuTTY](https://www.putty.org) in Windows as your [SSH] client, then you might want to take a quick -read on [using PuTTY in Windows][Using-SSH-In-Putty]. +If you're using [PuTTY](https://www.putty.org) in Windows as your [SSH](https://en.wikipedia.org/wiki/Secure_Shell) client, then you might want to take a quick +read on [using PuTTY in Windows](https://mediatemple.net/community/products/dv/204404604/using-ssh-in-putty-). ### Updating GitLab @@ -412,31 +412,16 @@ Check out our other [Technical Articles](../../articles/index.md) or browse the ### Useful links -- [GitLab Community Edition][CE] -- [GitLab Enterprise Edition][EE] -- [Microsoft Azure][Azure] - - [Azure - Free Account FAQ][Azure-Free-Account-FAQ] +- [GitLab Community Edition](https://about.gitlab.com/features/) +- [GitLab Enterprise Edition](https://about.gitlab.com/features/#ee-starter) +- [Microsoft Azure](https://azure.microsoft.com/en-us/) + - [Azure - Free Account FAQ](https://azure.microsoft.com/en-us/free/free-account-faq/) - [Azure - Marketplace](https://azuremarketplace.microsoft.com/en-us/marketplace/) - - [Azure Portal][Azure-Portal] - - [Azure - Pricing Calculator][Azure-Pricing-Calculator] + - [Azure Portal](https://portal.azure.com) + - [Azure - Pricing Calculator](https://azure.microsoft.com/en-us/pricing/calculator/) - [Azure - Troubleshoot SSH Connections to an Azure Linux VM](https://docs.microsoft.com/en-us/azure/virtual-machines/troubleshooting/troubleshoot-ssh-connection) - [Azure - Properly Shutdown an Azure VM](https://build5nines.com/properly-shutdown-azure-vm-to-save-money/) -- [SSH], [PuTTY](https://www.putty.org) and [Using SSH in PuTTY][Using-SSH-In-Putty] - -[Original-Blog-Post]: https://about.gitlab.com/blog/2016/07/13/how-to-setup-a-gitlab-instance-on-microsoft-azure/ "How to Set up a GitLab Instance on Microsoft Azure" -[CE]: https://about.gitlab.com/features/ -[EE]: https://about.gitlab.com/features/#ee-starter - -[Azure-Troubleshoot-Linux-VM]: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/troubleshoot-app-connection "Troubleshoot application connectivity issues on a Linux virtual machine in Azure" -[Azure-IP-Address-Types]: https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-ip-addresses-overview-arm "IP address types and allocation methods in Azure" -[Azure-How-To-Open-Ports]: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/nsg-quickstart-portal "How to open ports to a virtual machine with the Azure portal" -[Azure]: https://azure.microsoft.com/en-us/ -[Azure-Free-Account-FAQ]: https://azure.microsoft.com/en-us/free/free-account-faq/ -[Azure-Portal]: https://portal.azure.com -[Azure-Pricing-Calculator]: https://azure.microsoft.com/en-us/pricing/calculator/ - -[SSH]: https://en.wikipedia.org/wiki/Secure_Shell -[Using-SSH-In-Putty]: https://mediatemple.net/community/products/dv/204404604/using-ssh-in-putty- +- [SSH](https://en.wikipedia.org/wiki/Secure_Shell), [PuTTY](https://www.putty.org) and [Using SSH in PuTTY](https://mediatemple.net/community/products/dv/204404604/using-ssh-in-putty-) <!-- ## Troubleshooting diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index 9fcf6e6420c..433eeda00b1 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -5,7 +5,7 @@ type: howto # Installing GitLab on Google Cloud Platform -This guide will help you install GitLab on a [Google Cloud Platform (GCP)][gcp] instance. +This guide will help you install GitLab on a [Google Cloud Platform (GCP)](https://cloud.google.com/) instance. NOTE: **Alternative installation method:** Google provides a whitepaper for [deploying production-ready GitLab on @@ -19,7 +19,7 @@ There are only two prerequisites in order to install GitLab on GCP: 1. You need to have a Google account. 1. You need to sign up for the GCP program. If this is your first time, Google - gives you [$300 credit for free][freetrial] to consume over a 60-day period. + gives you [$300 credit for free](https://console.cloud.google.com/freetrial) to consume over a 60-day period. Once you have performed those two steps, you can [create a VM](#creating-the-vm). @@ -37,7 +37,7 @@ To deploy GitLab on GCP you first need to create a virtual machine:  -1. Click **Change** under Boot disk to select the size, type, and desired operating system. GitLab supports a [variety of linux operating systems][req], including Ubuntu and Debian. Click **Select** when finished. +1. Click **Change** under Boot disk to select the size, type, and desired operating system. GitLab supports a [variety of linux operating systems](../requirements.md), including Ubuntu and Debian. Click **Select** when finished.  @@ -85,7 +85,7 @@ here's how you configure GitLab to be aware of the change:  - In the future you might want to set up [connecting with an SSH key][ssh] + In the future you might want to set up [connecting with an SSH key](https://cloud.google.com/compute/docs/instances/connecting-to-instance) instead. 1. Edit the config file of Omnibus GitLab using your favorite text editor: @@ -114,13 +114,13 @@ here's how you configure GitLab to be aware of the change: ### Configuring HTTPS with the domain name Although not needed, it's strongly recommended to secure GitLab with a TLS -certificate. Follow the steps in the [Omnibus documentation][omni-ssl]. +certificate. Follow the steps in the [Omnibus documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### Configuring the email SMTP settings You need to configure the email SMTP settings correctly otherwise GitLab will not be able to send notification emails, like comments, and password changes. -Check the [Omnibus documentation][omni-smtp] how to do so. +Check the [Omnibus documentation](https://docs.gitlab.com/omnibus/settings/smtp.html#smtp-settings) how to do so. ## Further reading @@ -132,13 +132,6 @@ Kerberos, etc. Here are some documents you might be interested in reading: - [GitLab Pages configuration](../../administration/pages/index.md) - [GitLab Container Registry configuration](../../administration/packages/container_registry.md) -[freetrial]: https://console.cloud.google.com/freetrial "GCP free trial" -[gcp]: https://cloud.google.com/ "Google Cloud Platform" -[req]: ../requirements.md "GitLab hardware and software requirements" -[ssh]: https://cloud.google.com/compute/docs/instances/connecting-to-instance "Connecting to Linux Instances" -[omni-smtp]: https://docs.gitlab.com/omnibus/settings/smtp.html#smtp-settings "Omnibus GitLab SMTP settings" -[omni-ssl]: https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https "Omnibus GitLab enable HTTPS" - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/install/installation.md b/doc/install/installation.md index 740cd12b7fd..dd619e9e7b3 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -142,21 +142,31 @@ Starting with GitLab 12.0, Git is required to be compiled with `libpcre2`. Find out if that's the case: ```shell -ldd /usr/local/bin/git | grep pcre2 +ldd $(which git) | grep pcre2 ``` -The output should be similar to: +The output should contain `libpcre2-8.so.0`. -```plaintext -libpcre2-8.so.0 => /usr/lib/libpcre2-8.so.0 (0x00007f08461c3000) +Is the system packaged Git too old, or not compiled with pcre2? +Remove it: + +```shell +sudo apt-get remove git-core ``` -Is the system packaged Git too old, or not compiled with pcre2? Remove it and compile from source: +On Ubuntu, install Git from [its official PPA](https://git-scm.com/download/linux): ```shell -# Remove packaged Git -sudo apt-get remove git-core +# run as root! +add-apt-repository ppa:git-core/ppa +apt update +apt install git +# repeat libpcre2 check as above +``` + +On Debian, use the following compilation instructions: +```shell # Install dependencies sudo apt-get install -y libcurl4-openssl-dev libexpat1-dev gettext libz-dev libssl-dev build-essential @@ -180,7 +190,7 @@ make prefix=/usr/local all # Install into /usr/local/bin sudo make prefix=/usr/local install -# When editing config/gitlab.yml (Step 5), change the git -> bin_path to /usr/local/bin/git +# When editing config/gitlab.yml later, change the git -> bin_path to /usr/local/bin/git ``` For the [Custom Favicon](../user/admin_area/appearance.md#favicon) to work, GraphicsMagick @@ -205,7 +215,7 @@ The Ruby interpreter is required to run GitLab. **Note:** The current supported Ruby (MRI) version is 2.6.x. GitLab 12.2 dropped support for Ruby 2.5.x. -The use of Ruby version managers such as [RVM], [rbenv](https://github.com/rbenv/rbenv) or [chruby] with GitLab +The use of Ruby version managers such as [RVM](https://rvm.io/), [rbenv](https://github.com/rbenv/rbenv) or [chruby](https://github.com/postmodern/chruby) with GitLab in production, frequently leads to hard to diagnose problems. For example, GitLab Shell is called from OpenSSH, and having a version manager can prevent pushing and pulling over SSH. Version managers are not supported and we strongly @@ -360,10 +370,10 @@ use of extensions and concurrent index removal, you need at least PostgreSQL 9.2 ## 7. Redis -GitLab requires at least Redis 2.8. +GitLab requires at least Redis 5.0. -If you are using Debian 8 or Ubuntu 14.04 and up, you can simply install -Redis 2.8 with: +If you are using Debian 10 or Ubuntu 20.04 and up, you can install +Redis 5.0 with: ```shell sudo apt-get install redis-server @@ -966,7 +976,7 @@ If you want to switch back to Unicorn, follow these steps: sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb ``` -1. Edit the system `init.d` script to set the `USE_UNICORN=1` flag. If you have `/etc/default/gitlab`, then you should edit it instead. +1. Edit the system `init.d` script and set `USE_WEB_SERVER="unicorn"`. If you have `/etc/default/gitlab`, then you should edit it instead. 1. Restart GitLab. ### Using Sidekiq instead of Sidekiq Cluster @@ -1035,6 +1045,3 @@ On RedHat/CentOS: ```shell sudo yum groupinstall 'Development Tools' ``` - -[RVM]: https://rvm.io/ "RVM Homepage" -[chruby]: https://github.com/postmodern/chruby "chruby on GitHub" diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index a71d839e742..fd81b7f6caf 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -14,7 +14,7 @@ for details. ## Introduction [OpenShift Origin](https://www.okd.io/) (**Note:** renamed to OKD in Aug 2018) is an open source container application -platform created by [RedHat], based on [Kubernetes](https://kubernetes.io/) and [Docker]. That means +platform created by [RedHat](https://www.redhat.com/en), based on [Kubernetes](https://kubernetes.io/) and [Docker](https://www.docker.com). That means you can host your own PaaS for free and almost with no hassle. In this tutorial, we will see how to deploy GitLab in OpenShift using GitLab's @@ -34,16 +34,16 @@ offered by the OpenShift developers and managed by Vagrant. If you haven't done already, go ahead and install the following components as they are essential to test OpenShift easily: -- [VirtualBox] -- [Vagrant] +- [VirtualBox](https://www.virtualbox.org/wiki/Downloads) +- [Vagrant](https://www.vagrantup.com/downloads.html) - [OpenShift Client](https://docs.okd.io/3.11/cli_reference/get_started_cli.html) (`oc` for short) It is also important to mention that for the purposes of this tutorial, the latest Origin release is used: -- **oc** `v1.3.0` (must be [installed][oc-gh] locally on your computer) -- **OpenShift** `v1.3.0` (is pre-installed in the [VM image][vm-new]) -- **Kubernetes** `v1.3.0` (is pre-installed in the [VM image][vm-new]) +- **oc** `v1.3.0` (must be [installed](https://github.com/openshift/origin/releases/tag/v1.3.0) locally on your computer) +- **OpenShift** `v1.3.0` (is pre-installed in the [VM image](https://app.vagrantup.com/openshift/boxes/origin-all-in-one)) +- **Kubernetes** `v1.3.0` (is pre-installed in the [VM image](https://app.vagrantup.com/openshift/boxes/origin-all-in-one)) >**Note:** If you intend to deploy GitLab on a production OpenShift cluster, there are some @@ -302,7 +302,7 @@ template: - `gitlab-ce-postgresql` While PostgreSQL and Redis are bundled in Omnibus GitLab, the template is using -separate images as you can see from [this line][line] in the template. +separate images as you can see from [this line](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/658c065c8d022ce858dd63eaeeadb0b2ddc8deea/docker/openshift-template.json#L239) in the template. The predefined values have been calculated for the purposes of testing out GitLab in the all-in-one VM. You don't need to change anything here, hit @@ -371,7 +371,7 @@ running scaled to 2. Upping the GitLab pods is actually like adding new application servers to your cluster. You can see how that would work if you didn't use GitLab with -OpenShift by following the [HA documentation][ha] for the application servers. +OpenShift by following the [HA documentation](../../administration/high_availability/gitlab.md) for the application servers. Bare in mind that you may need more resources (CPU, RAM, disk space) when you scale up. If a pod is in pending state for too long, you can navigate to @@ -505,14 +505,3 @@ And remember that in this tutorial we just scratched the surface of what Origin is capable of. As always, you can refer to the detailed [documentation](https://docs.okd.io) to learn more about deploying your own OpenShift PaaS and managing your applications with the ease of containers. - -[RedHat]: https://www.redhat.com/en "RedHat website" -[vm-new]: https://app.vagrantup.com/openshift/boxes/origin-all-in-one "Official OpenShift Vagrant box on Vagrant Cloud" -[template]: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/docker/openshift-template.json "OpenShift template for GitLab" -[Docker]: https://www.docker.com "Docker website" -[VirtualBox]: https://www.virtualbox.org/wiki/Downloads "VirtualBox downloads" -[Vagrant]: https://www.vagrantup.com/downloads.html "Vagrant downloads" -[old-post]: https://blog.openshift.com/deploy-gitlab-openshift/ "Old post - Deploy GitLab on OpenShift" -[line]: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/658c065c8d022ce858dd63eaeeadb0b2ddc8deea/docker/openshift-template.json#L239 "GitLab - OpenShift template" -[oc-gh]: https://github.com/openshift/origin/releases/tag/v1.3.0 "OpenShift Origin 1.3.0 release on GitHub" -[ha]: ../../administration/high_availability/gitlab.md "Documentation - GitLab High Availability" diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index 595304e27e2..f8ff8e75c4d 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -10,7 +10,7 @@ be installed under a relative URL, for example `https://example.com/gitlab`. This document describes how to run GitLab under a relative URL for installations from source. If you are using an Omnibus package, -[the steps are different][omnibus-rel]. Use this guide along with the +[the steps are different](https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab). Use this guide along with the [installation guide](installation.md) if you are installing GitLab for the first time. @@ -30,7 +30,7 @@ serve GitLab under a relative URL is: - `/home/git/gitlab-shell/config.yml` - `/etc/default/gitlab` -After all the changes you need to recompile the assets and [restart GitLab]. +After all the changes you need to recompile the assets and [restart GitLab](../administration/restart_gitlab.md#installations-from-source). ## Relative URL requirements @@ -112,7 +112,7 @@ Make sure to follow all steps below: If you are using a custom init script, make sure to edit the above GitLab Workhorse setting as needed. -1. [Restart GitLab][] for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. ## Disable relative URL in GitLab @@ -123,9 +123,6 @@ To disable the relative URL: 1. Follow the same as above starting from 2. and set up the GitLab URL to one that doesn't contain a relative path. -[omnibus-rel]: https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab "How to set up relative URL in Omnibus GitLab" -[restart gitlab]: ../administration/restart_gitlab.md#installations-from-source "How to restart GitLab" - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/install/requirements.md b/doc/install/requirements.md index f78525659f2..09ad5f9afd7 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -11,15 +11,15 @@ as the hardware requirements that are needed to install and use GitLab. ### Supported Linux distributions -- Ubuntu -- Debian -- CentOS -- openSUSE +- Ubuntu (16.04/18.04) +- Debian (8/9/10) +- CentOS (6/7/8) +- openSUSE (Leap 15.1/Enterprise Server 12.2) - Red Hat Enterprise Linux (please use the CentOS packages and instructions) - Scientific Linux (please use the CentOS packages and instructions) - Oracle Linux (please use the CentOS packages and instructions) -For the installations options, see [the main installation page](README.md). +For the installation options, see [the main installation page](README.md). ### Unsupported Linux distributions and Unix-like operating systems @@ -68,10 +68,14 @@ GitLab uses [webpack](https://webpack.js.org/) to compile frontend assets, which version of Node.js 10.13.0. You can check which version you are running with `node -v`. If you are running -a version older than `v10.13.0`, you need to update to a newer version. You +a version older than `v10.13.0`, you need to update it to a newer version. You can find instructions to install from community maintained packages or compile from source at the [Node.js website](https://nodejs.org/en/download/). +## Redis versions + +GitLab requires Redis 5.0+. Beginning in GitLab 13.0, lower versions are not supported. + ## Hardware requirements ### Storage @@ -95,7 +99,7 @@ This is the recommended minimum hardware for a handful of example GitLab user ba - 4 cores supports up to 500 users - 8 cores supports up to 1,000 users - 32 cores supports up to 5,000 users -- More users? Run it high-availability on [multiple application servers](https://about.gitlab.com/solutions/high-availability/) +- More users? Consult the [reference architectures page](../administration/reference_architectures/index.md) ### Memory @@ -112,7 +116,7 @@ errors during usage. - 16GB RAM supports up to 500 users - 32GB RAM supports up to 1,000 users - 128GB RAM supports up to 5,000 users -- More users? Run it high-availability on [multiple application servers](https://about.gitlab.com/solutions/high-availability/) +- More users? Consult the [reference architectures page](../administration/reference_architectures/index.md) We recommend having at least [2GB of swap on your server](https://askubuntu.com/a/505344/310789), even if you currently have enough available RAM. Having swap will help reduce the chance of errors occurring @@ -122,7 +126,7 @@ available when needed. Our [Memory Team](https://about.gitlab.com/handbook/engineering/development/enablement/memory/) is actively working to reduce the memory requirement. -NOTE: **Note:** The 25 workers of Sidekiq will show up as separate processes in your process overview (such as `top` or `htop`) but they share the same RAM allocation since Sidekiq is a multithreaded application. Please see the section below about Unicorn workers for information about how many you need of those. +NOTE: **Note:** The 25 workers of Sidekiq will show up as separate processes in your process overview (such as `top` or `htop`) but they share the same RAM allocation since Sidekiq is a multithreaded application. Please see the section below about Unicorn workers for information about how many you need for those. ## Database @@ -139,9 +143,12 @@ MySQL/MariaDB are advised to [migrate to PostgreSQL](../update/mysql_to_postgres ### PostgreSQL Requirements -As of GitLab 10.0, PostgreSQL 9.6 or newer is required, and earlier versions are -not supported. We highly recommend users to use PostgreSQL 9.6 as this -is the PostgreSQL version used for development and testing. +We highly recommend users to use the minimum PostgreSQL versions specified below as these are the versions used for development and testing. + +GitLab version | Minimum PostgreSQL version +-|- +10.0 | 9.6 +12.10 | 11 Users using PostgreSQL must ensure the `pg_trgm` extension is loaded into every GitLab database. This extension can be enabled (using a PostgreSQL super user) @@ -167,7 +174,7 @@ If you are using [GitLab Geo](../development/geo.md): - The [tracking database](../development/geo.md#using-the-tracking-database) requires the - [postgres_fdw](https://www.postgresql.org/docs/9.6/postgres-fdw.html) + [postgres_fdw](https://www.postgresql.org/docs/11/postgres-fdw.html) extension. ```sql @@ -180,28 +187,50 @@ For most instances we recommend using: (CPU cores * 1.5) + 1 = Unicorn workers. For example a node with 4 cores would have 7 Unicorn workers. For all machines that have 2GB and up we recommend a minimum of three Unicorn workers. -If you have a 1GB machine we recommend to configure only two Unicorn workers to prevent excessive swapping. +If you have a 1GB machine we recommend to configure only two Unicorn workers to prevent excessive +swapping. + +As long as you have enough available CPU and memory capacity, it's okay to increase the number of +Unicorn workers and this will usually help to reduce the response time of the applications and +increase the ability to handle parallel requests. + +To change the Unicorn workers when you have the Omnibus package (which defaults to the +recommendation above) please see [the Unicorn settings in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/unicorn.html). + +## Puma settings + +The recommended settings for Puma are determined by the infrastructure on which it's running. +Omnibus GitLab defaults to the recommended Puma settings. Regardless of installation method, you can +tune the Puma settings. -As long as you have enough available CPU and memory capacity, it's okay to increase the number of Unicorn workers and this will usually help to reduce the response time of the applications and increase the ability to handle parallel requests. +If you're using Omnibus GitLab, see [Puma settings](https://docs.gitlab.com/omnibus/settings/puma.html) +for instructions on changing the Puma settings. -To change the Unicorn workers when you have the Omnibus package (which defaults to the recommendation above) please see [the Unicorn settings in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/unicorn.html). +### Puma workers -## Puma Workers +The recommended number of workers is calculated as the highest of the following: -For most instances we recommend using: max(CPU cores * 0.9, 2) = Puma workers. -For example a node with 4 cores would have 3 Puma workers. +- `2` +- Number of CPU cores - 1 -For all machines that have 4GB and up we recommend a minimum of three Puma workers. -If you have a 2GB machine we recommend to configure only one Puma worker to prevent excessive swapping. +For example a node with 4 cores should be configured with 3 Puma workers. -By default each Puma worker runs with 4 threads. We do not recommend setting more, -due to how [Ruby MRI multi-threading](https://en.wikipedia.org/wiki/Global_interpreter_lock) works. +You can increase the number of Puma workers, providing enough CPU and memory capacity is available. +A higher number of Puma workers will usually help to reduce the response time of the application +and increase the ability to handle parallel requests. You must perform testing to verify the +optimal settings for your infrastructure. -For cases when you have to use [Legacy Rugged code](../development/gitaly.md#legacy-rugged-code) it is recommended to set number of threads to 1. +### Puma threads -As long as you have enough available CPU and memory capacity, it's okay to increase the number of Puma workers and this will usually help to reduce the response time of the applications and increase the ability to handle parallel requests. +The recommended number of threads is dependent on several factors, including total memory, and use +of [legacy Rugged code](../development/gitaly.md#legacy-rugged-code). -To change the Puma workers when you have the Omnibus package (which defaults to the recommendation above) please see [the Puma settings in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/puma.html). +- If the operating system has a maximum 2 GB of memory, the recommended number of threads is `1`. + A higher value will result in excess swapping, and decrease performance. +- If legacy Rugged code is in use, the recommended number of threads is `1`. +- In all other cases, the recommended number of threads is `4`. We do not recommend setting this +higher, due to how [Ruby MRI multi-threading](https://en.wikipedia.org/wiki/Global_interpreter_lock) +works. ## Redis and Sidekiq diff --git a/doc/integration/README.md b/doc/integration/README.md index bba38922b9d..f06cb5e4bb6 100644 --- a/doc/integration/README.md +++ b/doc/integration/README.md @@ -83,7 +83,7 @@ at Super User also has relevant information. **Omnibus Trusted Chain** [Install the self signed certificate or custom certificate authorities](https://docs.gitlab.com/omnibus/common_installation_problems/README.html#using-self-signed-certificate-or-custom-certificate-authorities) -in to GitLab Omnibus. +in to Omnibus GitLab. It is enough to concatenate the certificate to the main trusted certificate however it may be overwritten during upgrades: diff --git a/doc/integration/azure.md b/doc/integration/azure.md index 75469fcbb98..809a3b703fc 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -22,7 +22,7 @@ To enable the Microsoft Azure OAuth2 OmniAuth provider you must register your ap 1. Add a "Reply URL" pointing to the Azure OAuth callback of your GitLab installation (e.g. `https://gitlab.mycompany.com/users/auth/azure_oauth2/callback`). -1. Create a "Client secret" by selecting a duration, the secret will be generated as soon as you click the "Save" button in the bottom menu.. +1. Create a "Client secret" by selecting a duration, the secret will be generated as soon as you click the "Save" button in the bottom menu. 1. Note the "CLIENT ID" and the "CLIENT SECRET". diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index fcd1c03a556..ae9d5bd8f60 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -48,12 +48,12 @@ For indexing Git repository data, GitLab uses an [indexer written in Go](https:/ The way you install the Go indexer depends on your version of GitLab: -- For GitLab Omnibus 11.8 and above, see [GitLab Omnibus](#gitlab-omnibus). -- For installations from source or older versions of GitLab Omnibus, install the indexer [From Source](#from-source). +- For Omnibus GitLab 11.8 and above, see [Omnibus GitLab](#omnibus-gitlab). +- For installations from source or older versions of Omnibus GitLab, install the indexer [From Source](#from-source). -### GitLab Omnibus +### Omnibus GitLab -Since GitLab 11.8 the Go indexer is included in GitLab Omnibus. +Since GitLab 11.8 the Go indexer is included in Omnibus GitLab. The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/issues/6481). ### From source @@ -152,8 +152,8 @@ The following Elasticsearch settings are available: | `AWS Access Key` | The AWS access key. | | `AWS Secret Access Key` | The AWS secret access key. | | `Maximum field length` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-field-length). | -| `Maximum bulk request size (MiB)` | Repository indexing uses the Elasticsearch bulk request API. This setting determines the maximum size of an individual bulk request during these operations. | -| `Bulk request concurrency` | Each repository indexing operation may submit bulk requests in parallel. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. | +| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by GitLab's Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch’s Bulk API. This setting works in tandem with the Bulk request concurrency setting (see below) and needs to accomodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Bulk request concurrency` | The Bulk request concurrency indicates how many of GitLab's Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch’s Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting works in tandem with the Maximum bulk request size setting (see above) and needs to accomodate the resource constraints of both the Elasticsearch host(s) and the host(s) running GitLab's Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | ### Limiting namespaces and projects @@ -174,7 +174,7 @@ If no namespaces or projects are selected, no Elasticsearch indexing will take p CAUTION: **Warning**: If you have already indexed your instance, you will have to regenerate the index in order to delete all existing data -for filtering to work correctly. To do this run the Rake tasks `gitlab:elastic:create_empty_index` and +for filtering to work correctly. To do this run the Rake tasks `gitlab:elastic:recreate_index` and `gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list will delete the data from the Elasticsearch index as expected. @@ -235,6 +235,8 @@ To index via the Admin Area: ### Indexing through Rake tasks +Indexing can be performed using Rake tasks. + #### Indexing small instances CAUTION: **Warning**: @@ -400,43 +402,32 @@ or creating [extra Sidekiq processes](../administration/operations/extra_sidekiq For repository and snippet files, GitLab will only index up to 1 MiB of content, in order to avoid indexing timeouts. -## GitLab Elasticsearch Rake Tasks - -There are several Rake tasks available to you via the command line: - -- [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - This is a wrapper task. It does the following: - - `sudo gitlab-rake gitlab:elastic:create_empty_index` - - `sudo gitlab-rake gitlab:elastic:clear_index_status` - - `sudo gitlab-rake gitlab:elastic:index_projects` - - `sudo gitlab-rake gitlab:elastic:index_snippets` -- [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - This iterates over all projects and queues Sidekiq jobs to index them in the background. -- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - This determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. -- [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - This generates an empty index on the Elasticsearch side, deleting the existing one if present. -- [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - This deletes all instances of IndexStatus for all projects. -- [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - This removes the GitLab index on the Elasticsearch instance. -- [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - Does the same thing as `sudo gitlab-rake gitlab:elastic:create_empty_index` -- [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - Performs an Elasticsearch import that indexes the snippets data. -- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - Displays which projects are not indexed. -- [`sudo gitlab-rake gitlab:elastic:reindex_to_another_cluster[<SOURCE_CLUSTER_URL>,<DESTINATION_CLUSTER_URL>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) - - Creates a new index in the destination cluster from the source index using - Elasticsearch "reindex from remote", where the source index is copied to the - destination. This is useful when migrating to a new cluster because it should be - quicker than reindexing via GitLab. - - NOTE: **Note:** - Your source cluster must be whitelisted in your destination cluster's Elasticsearch - settings. See [Reindex from remote](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#reindex-from-remote). - -### Environment Variables +## GitLab Elasticsearch Rake tasks + +Rake tasks are available to: + +- [Build and install](#building-and-installing) the indexer. +- Delete indexes when [disabling Elasticsearch](#disabling-elasticsearch). +- [Add GitLab data](#adding-gitlabs-data-to-the-elasticsearch-index) to an index. + +The following are some available Rake tasks: + +| Task | Description | +|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | +| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects and queues Sidekiq jobs to index them in the background. | +| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | +| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. | +| [`sudo gitlab-rake gitlab:elastic:create_empty_index[<INDEX_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates an empty index on the Elasticsearch side only if it doesn't already exist. | +| [`sudo gitlab-rake gitlab:elastic:delete_index[<INDEX_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab index on the Elasticsearch instance. | +| [`sudo gitlab-rake gitlab:elastic:recreate_index[<INDEX_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index[<INDEX_NAME>]` and `gitlab:elastic:create_empty_index[<INDEX_NAME>]`. | +| [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. | +| [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. | + +NOTE: **Note:** +The `INDEX_NAME` parameter is optional and will use the default index name from the current `RAILS_ENV` if not set. + +### Environment variables In addition to the Rake tasks, there are some environment variables that can be used to modify the process: @@ -456,7 +447,7 @@ Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! ``` -## Elasticsearch Index Scopes +## Elasticsearch index scopes When performing a search, the GitLab index will use the following scopes: @@ -506,6 +497,8 @@ However, some larger installations may wish to tune the merge policy settings: ## Troubleshooting +### Common issues + Here are some common pitfalls and how to overcome them: - **How can I verify my GitLab instance is using Elasticsearch?** @@ -631,9 +624,13 @@ Here are some common pitfalls and how to overcome them: Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" ``` - You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticseach Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). + You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). Once you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](#gitlab-elasticsearch-rake-tasks)) and [reindex the content of your instance](#adding-gitlabs-data-to-the-elasticsearch-index). +### Low level troubleshooting + +There is more [low level troubleshooting documentation](../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. + ### Known Issues - **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/gitlab-org/gitlab/issues/10693)** diff --git a/doc/integration/github.md b/doc/integration/github.md index c957fc24eb8..68971c510d5 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -138,11 +138,8 @@ You will also need to disable Git SSL verification on the server hosting GitLab. git config --global http.sslVerify false ``` -For the changes to take effect, [reconfigure GitLab] if you installed -via Omnibus, or [restart GitLab] if you installed from source. - -[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source +For the changes to take effect, [reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) if you installed +via Omnibus, or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) if you installed from source. ## Troubleshooting diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 4e62f6389c8..f423863ce8e 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -24,9 +24,8 @@ GitLab.com will generate an application ID and secret key for you to use. 1. Select **Save application**. -1. You should now see a **Application Id** and **Secret** near the top right of the page (see screenshot). - Keep this page open as you continue configuration. -  +1. You should now see an **Application ID** and **Secret**. Keep this page open as you continue + configuration. 1. On your GitLab server, open the configuration file. diff --git a/doc/integration/img/gitlab_app.png b/doc/integration/img/gitlab_app.png Binary files differdeleted file mode 100644 index 228e8a01305..00000000000 --- a/doc/integration/img/gitlab_app.png +++ /dev/null diff --git a/doc/integration/img/jenkins_gitlab_service_settings.png b/doc/integration/img/jenkins_gitlab_service_settings.png Binary files differdeleted file mode 100644 index 5a12e9cb39a..00000000000 --- a/doc/integration/img/jenkins_gitlab_service_settings.png +++ /dev/null diff --git a/doc/integration/jenkins.md b/doc/integration/jenkins.md index 5514b756c00..485ca67c9fc 100644 --- a/doc/integration/jenkins.md +++ b/doc/integration/jenkins.md @@ -1,119 +1,163 @@ # Jenkins CI service **(STARTER)** ->**Note:** -In GitLab 8.3, Jenkins integration using the -[GitLab Hook Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) -was deprecated in favor of the -[GitLab Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Plugin). -The deprecated integration has been renamed to [Jenkins CI (Deprecated)](jenkins_deprecated.md) in the -integration settings. We may remove this in a future release and recommend -using the new 'Jenkins CI' integration instead which is described in this -document. - -## Overview - -[Jenkins](https://jenkins.io/) is a great Continuous Integration tool, similar to our built-in -[GitLab CI/CD](../ci/README.md). - -GitLab's Jenkins integration allows you to trigger a Jenkins build when you -push code to a repository, or when a merge request is created. Additionally, -it shows the pipeline status on merge requests widgets and on the project's home page. - -Videos are also available on [GitLab workflow with Jira issues and Jenkins pipelines](https://youtu.be/Jn-_fyra7xQ) -and [Migrating from Jenkins to GitLab](https://www.youtube.com/watch?v=RlEVGOpYF5Y). - -## Use cases - -- Suppose you are new to GitLab, and want to keep using Jenkins until you prepare - your projects to build with [GitLab CI/CD](../ci/README.md). You set up the - integration between GitLab and Jenkins, then you migrate to GitLab CI/CD later. While - you organize yourself and your team to onboard GitLab, you keep your pipelines - running with Jenkins, but view the results in your project's repository in GitLab. -- Your team uses [Jenkins Plugins](https://plugins.jenkins.io/) for other proceedings, - therefore, you opt for keep using Jenkins to build your apps. Show the results of your - pipelines directly in GitLab. +NOTE: **Note:** +This documentation focuses only on how to **configure** a Jenkins *integration* with +GitLab. Learn how to **migrate** from Jenkins to GitLab CI/CD in our +[Migrating from Jenkins](../ci/jenkins/index.md) documentation. + +From GitLab, you can trigger a Jenkins build when you push code to a repository, or when a merge +request is created. In return, Jenkins shows the pipeline status on merge requests widgets and +on the GitLab project's home page. + +To better understand GitLab's Jenkins integration, watch the following video: + +- [GitLab workflow with Jira issues and Jenkins pipelines](https://youtu.be/Jn-_fyra7xQ) +Use the Jenkins integration with GitLab when: + +- You plan to migrate your CI from Jenkins to [GitLab CI/CD](../ci/README.md) in the future, but +need an interim solution. +- You're invested in [Jenkins Plugins](https://plugins.jenkins.io/) and choose to keep using Jenkins +to build your apps. For a real use case, read the blog post [Continuous integration: From Jenkins to GitLab using Docker](https://about.gitlab.com/blog/2017/07/27/docker-my-precious/). -NOTE: **Moving from a traditional CI plug-in to a single application for the entire software development lifecycle can decrease hours spent on maintaining toolchains by 10% or more.** -Visit the ['GitLab vs. Jenkins' comparison page](https://about.gitlab.com/devops-tools/jenkins-vs-gitlab.html) to learn how our built-in CI compares to Jenkins. +Moving from a traditional CI plug-in to a single application for the entire software development +life cycle can decrease hours spent on maintaining toolchains by 10% or more. For more details, see +the ['GitLab vs. Jenkins' comparison page](https://about.gitlab.com/devops-tools/jenkins-vs-gitlab.html). -## Requirements +## Configure GitLab integration with Jenkins -- [Jenkins GitLab Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Plugin) -- [Jenkins Git Plugin](https://wiki.jenkins.io/display/JENKINS/Git+Plugin) -- Git clone access for Jenkins from the GitLab repository -- GitLab API access to report build status +GitLab's Jenkins integration requires installation and configuration in both GitLab and Jenkins. +In GitLab, you need to grant Jenkins access to the relevant projects. In Jenkins, you need to +install and configure several plugins. -## Configure GitLab users +### GitLab requirements -Create a user or choose an existing user that Jenkins will use to interact -through the GitLab API. This user will need to be a global Admin or added -as a member to each Group/Project. Developer permission is required for reporting -build status. This is because a successful build status can trigger a merge -when 'Merge when pipeline succeeds' feature is used. Some features of the GitLab -Plugin may require additional privileges. For example, there is an option to -accept a merge request if the build is successful. Using this feature would -require developer, maintainer or owner-level permission. +- [Grant Jenkins permission to GitLab project](#grant-jenkins-access-to-gitlab-project) +- [Configure GitLab API access](#configure-gitlab-api-access) +- [Configure the GitLab project](#configure-the-gitlab-project) -Copy the private API token from **Profile Settings -> Account**. You will need this -when configuring the Jenkins server later. +### Jenkins requirements -## Configure the Jenkins server +- [Configure the Jenkins server](#configure-the-jenkins-server) +- [Configure the Jenkins project](#configure-the-jenkins-project) -Install [Jenkins GitLab Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Plugin) -and [Jenkins Git Plugin](https://wiki.jenkins.io/display/JENKINS/Git+Plugin). +## Grant Jenkins access to GitLab project -Go to Manage Jenkins -> Configure System and scroll down to the 'GitLab' section. -Enter the GitLab server URL in the 'GitLab host URL' field and paste the API token -copied earlier in the 'API Token' field. +Grant a GitLab user access to the select GitLab projects. -For more information, see GitLab Plugin documentation about -[Jenkins-to-GitLab authentication](https://github.com/jenkinsci/gitlab-plugin#jenkins-to-gitlab-authentication) +1. Create a new GitLab user, or choose an existing GitLab user. - + This account will be used by Jenkins to access the GitLab projects. We recommend creating a GitLab + user for only this purpose. If you use a person's account, and their account is deactivated or + deleted, the GitLab-Jenkins integration will stop working. -## Configure a Jenkins project +1. Grant the user permission to the GitLab projects. -Follow the GitLab Plugin documentation about [Jenkins Job Configuration](https://github.com/jenkinsci/gitlab-plugin#jenkins-job-configuration). + If you're integrating Jenkins with many GitLab projects, consider granting the user global + Admin permission. Otherwise, add the user to each project, and grant Developer permission. -NOTE: **Note:** -Be sure to include the steps about [Build status configuration](https://github.com/jenkinsci/gitlab-plugin#build-status-configuration). -The 'Publish build status to GitLab' post-build step is required to view -Jenkins build status in GitLab Merge Requests. +## Configure GitLab API access -## Configure a GitLab project +Create a personal access token to authorize Jenkins' access to GitLab. -Create a new GitLab project or choose an existing one. Then, go to **Integrations -> -Jenkins CI**. +1. Log in to GitLab as the user to be used with Jenkins. +1. Click your avatar, then **Settings. +1. Click **Access Tokens** in the sidebar. +1. Create a personal access token with the **API** scope checkbox checked. For more details, see + [Personal access tokens](../user/profile/personal_access_tokens.md). +1. Record the personal access token's value, because it's required in [Configure the Jenkins server](#configure-the-jenkins-server). + +## Configure the Jenkins server -Check the 'Active' box. Select whether you want GitLab to trigger a build -on push, Merge Request creation, tag push, or any combination of these. We -recommend unchecking 'Merge Request events' unless you have a specific use-case -that requires re-building a commit when a merge request is created. With 'Push -events' selected, GitLab will build the latest commit on each push and the build -status will be displayed in the merge request. +Install and configure the Jenkins plugins. Both plugins must be installed and configured to +authorize the connection to GitLab. -Enter the Jenkins URL and Project name. The project name should be URL-friendly -where spaces are replaced with underscores. To be safe, copy the project name -from the URL bar of your browser while viewing the Jenkins project. +1. On the Jenkins server, go to **Manage Jenkins > Manage Plugins**. +1. Install the [Jenkins GitLab Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Plugin). +1. Go to **Manage Jenkins > Configure System**. +1. In the **GitLab** section, check the **Enable authentication for ‘/project’ end-point** checkbox. +1. Click **Add**, then choose **Jenkins Credential Provider**. +1. Choose **GitLab API token** as the token type. +1. Enter the GitLab personal access token's value in the **API Token** field and click **Add**. +1. Enter the GitLab server's URL in the **GitLab host URL** field. +1. Click **Test Connection**, ensuring the connection is successful before proceeding. -Optionally, enter a username and password if your Jenkins server requires -authentication. +For more information, see GitLab Plugin documentation about +[Jenkins-to-GitLab authentication](https://github.com/jenkinsci/gitlab-plugin#jenkins-to-gitlab-authentication). + + - +## Configure the Jenkins project + +Set up the Jenkins project you’re going to run your build on. + +1. On your Jenkins instance, go to **New Item**. +1. Enter the project's name. +1. Choose between **Freestyle** or **Pipeline** and click **OK**. + We recommend a Freestyle project, because the Jenkins plugin will update the build status on + GitLab. In a Pipeline project, you must configure a script to update the status on GitLab. +1. Choose your GitLab connection from the dropdown. +1. Check the **Build when a change is pushed to GitLab** checkbox. +1. Check the following checkboxes: + - **Accepted Merge Request Events** + - **Closed Merge Request Events** +1. Specify how build status is reported to GitLab: + - If you created a **Freestyle** project, in the **Post-build Actions** section, choose + **Publish build status to GitLab**. + - If you created a **Pipeline** project, you must use a Jenkins Pipeline script to update the status on + GitLab. + + Example Jenkins Pipeline script: + + ```groovy + pipeline { + agent any + + stages { + stage('gitlab') { + steps { + echo 'Notify GitLab' + updateGitlabCommitStatus name: 'build', state: 'pending' + updateGitlabCommitStatus name: 'build', state: 'success' + } + } + } + } + ``` + +## Configure the GitLab project + +Configure the GitLab integration with Jenkins. + +1. Create a new GitLab project or choose an existing one. +1. Go to **Settings > Integrations**, then select **Jenkins CI**. +1. Turn on the **Active** toggle. +1. Select the events you want GitLab to trigger a Jenkins build for: + - Push + - Merge request + - Tag push +1. Enter the **Jenkins URL**. +1. Enter the **Project name**. + + The project name should be URL-friendly, where spaces are replaced with underscores. To ensure + the project name is valid, copy it from your browser's address bar while viewing the Jenkins + project. +1. Enter the **Username** and **Password** if your Jenkins server requires + authentication. +1. Click **Test settings and save changes**. GitLab tests the connection to Jenkins. ## Plugin functional overview GitLab does not contain a database table listing commits. Commits are always -read from the repository directly. Therefore, it is not possible to retain the +read from the repository directly. Therefore, it's not possible to retain the build status of a commit in GitLab. This is overcome by requesting build information from the integrated CI tool. The CI tool is responsible for creating and storing build status for Commits and Merge Requests. ### Steps required to implement a similar integration ->**Note:** +**Note:** All steps are implemented using AJAX requests on the merge request page. 1. In order to display the build status in a merge request you must create a project service in GitLab. @@ -133,7 +177,7 @@ receive a build status update via the API. Either Jenkins was not properly configured or there was an error reporting the status via the API. 1. [Configure the Jenkins server](#configure-the-jenkins-server) for GitLab API access -1. [Configure a Jenkins project](#configure-a-jenkins-project), including the +1. [Configure the Jenkins project](#configure-the-jenkins-project), including the 'Publish build status to GitLab' post-build action. ### Merge Request event does not trigger a Jenkins Pipeline diff --git a/doc/integration/jenkins_deprecated.md b/doc/integration/jenkins_deprecated.md index af7f847f803..31bb271782a 100644 --- a/doc/integration/jenkins_deprecated.md +++ b/doc/integration/jenkins_deprecated.md @@ -6,6 +6,9 @@ was deprecated in favor of the [GitLab Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Plugin). Please use documentation for the new [Jenkins CI service](jenkins.md). +NOTE: **Note:** +This service was [removed in v13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/1600) + Integration includes: - Trigger Jenkins build after push to repo diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index 7753629c371..e9b1c9676bd 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -56,13 +56,16 @@ There are no special requirements if you are using GitLab.com. this would be `https://gitlab.com/login/oauth/callback`. NOTE: **Note**: - If using a GitLab version earlier than 11.3 the `Redirect URI` value should be `https://<your-gitlab-instance-domain>/-/jira/login/oauth/callback`. + If using a GitLab version earlier than 11.3, the `Redirect URI` must be + `https://<your-gitlab-instance-domain>/-/jira/login/oauth/callback`. If you want Jira + to have access to all projects, GitLab recommends an administrator creates the + Application.  - Check `api` in the Scopes section. -1. Click `Save application`. You will see the generated 'Application Id' and 'Secret' values. +1. Click `Save application`. You will see the generated 'Application ID' and 'Secret' values. Copy these values that you will use on the Jira configuration side. ## Jira Configuration @@ -164,7 +167,7 @@ Click the links to see your GitLab repository data. ## Limitations -- This integration is currently not supported on GitLab instances under a [relative url](https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab) (for example, `http://example.com/gitlab`). +- This integration is currently not supported on GitLab instances under a [relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab) (for example, `http://example.com/gitlab`). ## Changelog diff --git a/doc/integration/oauth_provider.md b/doc/integration/oauth_provider.md index 6c9b272f35b..fd1c21d725d 100644 --- a/doc/integration/oauth_provider.md +++ b/doc/integration/oauth_provider.md @@ -5,6 +5,8 @@ to sign in to other services. If you want to use: +- The [OAuth2](https://oauth.net/2/) protocol to access GitLab resources on user's behalf, + see [OAuth2 provider](../api/oauth2.md) - Other OAuth authentication service providers to sign in to GitLab, see the [OAuth2 client documentation](omniauth.md). - The related API, see [Applications API](../api/applications.md). diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 5634ad95cf7..2afdeccb764 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -51,7 +51,7 @@ that are in common for all providers that we need to consider. automatically create an account. It defaults to `false`. If `false` users must be created manually or they will not be able to sign in via OmniAuth. - `auto_link_ldap_user` can be used if you have [LDAP / ActiveDirectory](ldap.md) - integration enabled. It defaults to false. When enabled, users automatically + integration enabled. It defaults to `false`. When enabled, users automatically created through an OmniAuth provider will have their LDAP identity created in GitLab as well. - `block_auto_created_users` defaults to `true`. If `true` auto created users will be blocked by default and will have to be unblocked by an administrator before diff --git a/doc/integration/recaptcha.md b/doc/integration/recaptcha.md index 4f3140e4cb9..bf2e6568ea6 100644 --- a/doc/integration/recaptcha.md +++ b/doc/integration/recaptcha.md @@ -27,7 +27,7 @@ configuration variable: proxy_set_header X-GitLab-Show-Login-Captcha 1; ``` -In GitLab Omnibus, this can be configured via `/etc/gitlab/gitlab.rb`: +In Omnibus GitLab, this can be configured via `/etc/gitlab/gitlab.rb`: ```ruby nginx['proxy_set_headers'] = { 'X-GitLab-Show-Login-Captcha' => 1 } diff --git a/doc/integration/slash_commands.md b/doc/integration/slash_commands.md index 11abf3ca203..c73db32a42a 100644 --- a/doc/integration/slash_commands.md +++ b/doc/integration/slash_commands.md @@ -14,11 +14,11 @@ Taking the trigger term as `project-name`, the commands are: | ------- | ------ | | `/project-name help` | Shows all available slash commands | | `/project-name issue new <title> <shift+return> <description>` | Creates a new issue with title `<title>` and description `<description>` | -| `/project-name issue show <id>` | Shows the issue with id `<id>` | -| `/project-name issue close <id>` | Closes the issue with id `<id>` | +| `/project-name issue show <id>` | Shows the issue with ID `<id>` | +| `/project-name issue close <id>` | Closes the issue with ID `<id>` | | `/project-name issue search <query>` | Shows up to 5 issues matching `<query>` | | `/project-name issue move <id> to <project>` | Moves issue ID `<id>` to `<project>` | -| `/project-name issue comment <id> <shift+return> <comment>` | Adds a new comment to an issue with id `<id>` and comment body `<comment>` | +| `/project-name issue comment <id> <shift+return> <comment>` | Adds a new comment to an issue with ID `<id>` and comment body `<comment>` | | `/project-name deploy <from> to <to>` | Deploy from the `<from>` environment to the `<to>` environment | | `/project-name run <job name> <arguments>` | Execute [ChatOps](../ci/chatops/README.md) job `<job name>` on `master` | diff --git a/doc/integration/sourcegraph.md b/doc/integration/sourcegraph.md index 5da9dd1fbc9..90e9c7b9534 100644 --- a/doc/integration/sourcegraph.md +++ b/doc/integration/sourcegraph.md @@ -107,7 +107,7 @@ When visiting one of these views, you can now hover over a code reference to see ## Sourcegraph for GitLab.com -Sourcegraph powered code intelligence is avaialable for all public projects on GitLab.com. +Sourcegraph powered code intelligence is available for all public projects on GitLab.com. Support for private projects is currently not available for GitLab.com; follow the epic [&2201](https://gitlab.com/groups/gitlab-org/-/epics/2201) @@ -117,7 +117,7 @@ for updates. ### Sourcegraph isn't working -If you enabled Sourcegraph for your project but still it doesn't looklike it's working, it might be because Sourcegraph has not indexed theproject yet. You can check for Sourcegraph's availability of your project by visiting `https://sourcegraph.com/gitlab.com/<project-path>`replacing `<project-path>` with the path to your GitLab project. +If you enabled Sourcegraph for your project but still it doesn't look like it's working, it might be because Sourcegraph has not indexed the project yet. You can check for Sourcegraph's availability of your project by visiting `https://sourcegraph.com/gitlab.com/<project-path>`replacing `<project-path>` with the path to your GitLab project. ## Sourcegraph and Privacy diff --git a/doc/integration/vault.md b/doc/integration/vault.md index c29df9a24dc..0a6ced42925 100644 --- a/doc/integration/vault.md +++ b/doc/integration/vault.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- diff --git a/doc/monitoring/performance/influxdb_configuration.md b/doc/monitoring/performance/influxdb_configuration.md index ae5f4c7e9df..0bcb18bdfe7 100644 --- a/doc/monitoring/performance/influxdb_configuration.md +++ b/doc/monitoring/performance/influxdb_configuration.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../administration/monitoring/performance/influxdb_configuration.md' +redirect_to: '../../administration/monitoring/performance/prometheus.md' --- -This document was moved to [administration/monitoring/performance/influxdb_configuration](../../administration/monitoring/performance/influxdb_configuration.md). +Support for InfluxDB was removed in GitLab 13.0. Use [Prometheus](../../administration/monitoring/performance/prometheus.md) for performance monitoring. diff --git a/doc/monitoring/performance/influxdb_schema.md b/doc/monitoring/performance/influxdb_schema.md index 57fb74cb6cd..0bcb18bdfe7 100644 --- a/doc/monitoring/performance/influxdb_schema.md +++ b/doc/monitoring/performance/influxdb_schema.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../administration/monitoring/performance/influxdb_schema.md' +redirect_to: '../../administration/monitoring/performance/prometheus.md' --- -This document was moved to [administration/monitoring/performance/influxdb_schema](../../administration/monitoring/performance/influxdb_schema.md). +Support for InfluxDB was removed in GitLab 13.0. Use [Prometheus](../../administration/monitoring/performance/prometheus.md) for performance monitoring. diff --git a/doc/push_rules/push_rules.md b/doc/push_rules/push_rules.md index eca1f8c24a4..5685e848a33 100644 --- a/doc/push_rules/push_rules.md +++ b/doc/push_rules/push_rules.md @@ -9,11 +9,11 @@ regular expressions to reject pushes based on commit contents, branch names or f ## Overview -GitLab already offers [protected branches][protected-branches], but there are +GitLab already offers [protected branches](../user/project/protected_branches.md), but there are cases when you need some specific rules like preventing Git tag removal or enforcing a special format for commit messages. -Push rules are essentially [pre-receive Git hooks][hooks] that are easy to +Push rules are essentially [pre-receive Git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) that are easy to enable in a user-friendly interface. They are defined globally if you are an admin or per project so you can have different rules applied to different projects depending on your needs. @@ -43,11 +43,18 @@ you want the branches to start with a certain name because you have different GitLab CI/CD jobs (`feature`, `hotfix`, `docker`, `android`, etc.) that rely on the branch name. -Your developers however, don't always remember that policy, so they push -various branches and CI pipelines do not work as expected. By restricting the -branch names globally in Push Rules, you can now sleep without the anxiety -of your developers' mistakes. Every branch that doesn't match your push rule -will get rejected. +Your developers, however, don't always remember that policy, so they might push to +various branches, and CI pipelines might not work as expected. By restricting the +branch names globally in Push Rules, such mistakes are prevented. +Any branch name that doesn't match your push rule will get rejected. + +Note that the name of your default branch is always allowed, regardless of the branch naming +regular expression (regex) specified. GitLab is configured this way +because merges typically have the default branch as their target. +If you have other target branches, include them in your regex. (See [Enabling push rules](#enabling-push-rules)). + +The default branch also defaults to being a [protected branch](../user/project/protected_branches.md), +which already limits users from pushing directly. ### Custom Push Rules **(CORE ONLY)** @@ -61,7 +68,7 @@ See [server hooks](../administration/server_hooks.md) for more information. NOTE: **Note:** GitLab administrators can set push rules globally under **Admin Area > Push Rules** that all new projects will inherit. You can later -override them in a project's settings. +override them in a project's settings. They can be also set on a [group level](../user/group/index.md#group-push-rules-starter). 1. Navigate to your project's **Settings > Repository** and expand **Push Rules** 1. Set the rule you want @@ -74,7 +81,7 @@ The following options are available. | Removal of tags with `git push` | **Starter** 7.10 | Forbid users to remove Git tags with `git push`. Tags will still be able to be deleted through the web UI. | | Check whether author is a GitLab user | **Starter** 7.10 | Restrict commits by author (email) to existing GitLab users. | | Committer restriction | **Premium** 10.2 | GitLab will reject any commit that was not committed by the current authenticated user | -| Check whether commit is signed through GPG | **Premium** 10.1 | Reject commit when it is not signed through GPG. Read [signing commits with GPG][signing-commits]. | +| Check whether commit is signed through GPG | **Premium** 10.1 | Reject commit when it is not signed through GPG. Read [signing commits with GPG](../user/project/repository/gpg_signed_commits/index.md). | | Prevent committing secrets to Git | **Starter** 8.12 | GitLab will reject any files that are likely to contain secrets. Read [what files are forbidden](#prevent-pushing-secrets-to-the-repository). | | Restrict by commit message | **Starter** 7.10 | Only commit messages that match this regular expression are allowed to be pushed. Leave empty to allow any commit message. Uses multiline mode, which can be disabled using `(?-m)`. | | Restrict by commit message (negative match)| **Starter** 11.1 | Only commit messages that do not match this regular expression are allowed to be pushed. Leave empty to allow any commit message. Uses multiline mode, which can be disabled using `(?-m)`. | @@ -88,14 +95,15 @@ GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular ## Prevent pushing secrets to the repository -> [Introduced][ee-385] in [GitLab Starter][ee] 8.12. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/385) in [GitLab Starter](https://about.gitlab.com/pricing/) 8.12. -You can turn on a predefined blacklist of files which won't be allowed to be -pushed to a repository. +Secrets such as credential files, SSH private keys, and other files containing secrets should never be committed to source control. +GitLab allows you to turn on a predefined blacklist of files which won't be allowed to be +pushed to a repository, stopping those commits from reaching the remote repository. By selecting the checkbox *Prevent committing secrets to Git*, GitLab prevents pushes to the repository when a file matches a regular expression as read from -[`files_blacklist.yml`][list] (make sure you are at the right branch +[`files_blacklist.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/checks/files_blacklist.yml) (make sure you are at the right branch as your GitLab version when viewing this file). NOTE: **Note:** @@ -171,10 +179,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -[protected-branches]: ../user/project/protected_branches.md -[signing-commits]: ../user/project/repository/gpg_signed_commits/index.md -[ee-385]: https://gitlab.com/gitlab-org/gitlab/issues/385 -[list]: https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/checks/files_blacklist.yml -[hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks -[ee]: https://about.gitlab.com/pricing/ diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index 1670849016c..4bba4f2bc5a 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -15,25 +15,27 @@ GitLab Rake tasks are performed using: The following are available Rake tasks: -| Tasks | Description | -|:-------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------| -| [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. | -| [Clean up](cleanup.md) | Clean up unneeded items GitLab instances. | -| [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. | -| [Elasticsearch](../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) | Maintain Elasticsearch in a GitLab instance. | -| [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. | -| [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. | -| [Geo maintenance](../administration/raketasks/geo.md) **(PREMIUM ONLY)** | [Geo](../administration/geo/replication/index.md)-related maintenance. | -| [GitHub import](../administration/raketasks/github_import.md) | Retrieve and import repositories from GitHub. | -| [Import repositories](import.md) | Import bare repositories into your GitLab instance. | -| [Import large project exports](../development/import_project.md#importing-via-a-rake-task) | Import large GitLab [project exports](../user/project/settings/import_export.md). | -| [Integrity checks](../administration/raketasks/check.md) | Check the integrity of repositories, files, and LDAP. | -| [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap.md)-related tasks. | -| [List repositories](list_repos.md) | List of all GitLab-managed Git repositories on disk. | -| [Project import/export](../administration/raketasks/project_import_export.md) | Prepare for [project exports and imports](../user/project/settings/import_export.md). | -| [Repository storage](../administration/raketasks/storage.md) | List and migrate existing projects and attachments from legacy storage to hashed storage. | -| [Uploads migrate](../administration/raketasks/uploads/migrate.md) | Migrate uploads between storage local and object storage. | -| [Uploads sanitize](../administration/raketasks/uploads/sanitize.md) | Remove EXIF data from images uploaded to earlier versions of GitLab. | -| [User management](user_management.md) | Perform user management tasks. | -| [Webhooks administration](web_hooks.md) | Maintain project Webhooks. | -| [X509 signatures](x509_signatures.md) | Update x509 commit signatures, useful if certificate store has changed. | +| Tasks | Description | +|:----------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------| +| [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. | +| [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. | +| [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. | +| [Elasticsearch](../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) **(STARTER ONLY)** | Maintain Elasticsearch in a GitLab instance. | +| [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. | +| [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. | +| [Geo maintenance](../administration/raketasks/geo.md) **(PREMIUM ONLY)** | [Geo](../administration/geo/replication/index.md)-related maintenance. | +| [GitHub import](../administration/raketasks/github_import.md) | Retrieve and import repositories from GitHub. | +| [Import repositories](import.md) | Import bare repositories into your GitLab instance. | +| [Import large project exports](../development/import_project.md#importing-via-a-rake-task) | Import large GitLab [project exports](../user/project/settings/import_export.md). | +| [Integrity checks](../administration/raketasks/check.md) | Check the integrity of repositories, files, and LDAP. | +| [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap.md)-related tasks. | +| [List repositories](list_repos.md) | List of all GitLab-managed Git repositories on disk. | +| [Project import/export](../administration/raketasks/project_import_export.md) | Prepare for [project exports and imports](../user/project/settings/import_export.md). | +| [Sample Prometheus data](generate_sample_prometheus_data.md) | Generate sample Prometheus data. | +| [Repository storage](../administration/raketasks/storage.md) | List and migrate existing projects and attachments from legacy storage to hashed storage. | +| [Uploads migrate](../administration/raketasks/uploads/migrate.md) | Migrate uploads between storage local and object storage. | +| [Uploads sanitize](../administration/raketasks/uploads/sanitize.md) | Remove EXIF data from images uploaded to earlier versions of GitLab. | +| [User management](user_management.md) | Perform user management tasks. | +| [Webhooks administration](web_hooks.md) | Maintain project Webhooks. | +| [X.509 signatures](x509_signatures.md) | Update X.509 commit signatures, useful if certificate store has changed. | +| [Migrate Snippets to Git](migrate_snippets.md) | Migrate GitLab Snippets to Git repositories and show migration status | diff --git a/doc/raketasks/backup_hrz.png b/doc/raketasks/backup_hrz.png Binary files differdeleted file mode 100644 index 32690b2904c..00000000000 --- a/doc/raketasks/backup_hrz.png +++ /dev/null diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 24e4ed43543..3c81bab644e 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -1,6 +1,6 @@ -# Backing up and restoring GitLab +# Backing up and restoring GitLab **(CORE ONLY)** - +GitLab provides Rake tasks for backing up and restoring GitLab instances. An application data backup creates an archive file that contains the database, all repositories and all attachments. @@ -14,31 +14,26 @@ from one server to another is through backup restore. In order to be able to backup and restore, you need two essential tools installed on your system. -### Rsync +- **Rsync**: If you installed GitLab: + - Using the Omnibus package, you're all set. + - From source, make sure `rsync` is installed. For example: -If you installed GitLab: + ```shell + # Debian/Ubuntu + sudo apt-get install rsync -- Using the Omnibus package, you're all set. -- From source, make sure `rsync` is installed: + # RHEL/CentOS + sudo yum install rsync + ``` - ```shell - # Debian/Ubuntu - sudo apt-get install rsync +- **Tar**: Backup and restore tasks use `tar` under the hood to create and extract + archives. Ensure you have version 1.30 or above of `tar` available in your + system. To check the version, run: - # RHEL/CentOS - sudo yum install rsync + ```shell + tar --version ``` -### Tar - -Backup and restore tasks use `tar` under the hood to create and extract -archives. Ensure you have version 1.30 or above of `tar` available in your -system. To check the version, run: - -```shell -tar --version -``` - ## Backup timestamp NOTE: **Note:** @@ -56,9 +51,9 @@ available. For example, if the backup name is `1493107454_2018_04_25_10.6.4-ce_gitlab_backup.tar`, then the timestamp is `1493107454_2018_04_25_10.6.4-ce`. -## Creating a backup of the GitLab system +## Back up GitLab -GitLab provides a simple command line interface to backup your whole instance. +GitLab provides a simple command line interface to back up your whole instance. It backs up your: - Database @@ -142,12 +137,12 @@ Deleting tmp directories...[DONE] Deleting old backups... [SKIPPING] ``` -## Storing configuration files +### Storing configuration files -A backup performed by the [raketask GitLab provides](#creating-a-backup-of-the-gitlab-system) +The [backup Rake task](#back-up-gitlab) GitLab provides does **not** store your configuration files. The primary reason for this is that your database contains encrypted information for two-factor authentication, the CI/CD -'secure variables', etc. Storing encrypted information along with its key in the +'secure variables', and so on. Storing encrypted information along with its key in the same place defeats the purpose of using encryption in the first place. CAUTION: **Warning:** @@ -182,11 +177,11 @@ If you use Omnibus GitLab, see some additional information In the unlikely event that the secrets file is lost, see the [troubleshooting section](#when-the-secrets-file-is-lost). -## Backup options +### Backup options The command line tool GitLab provides to backup your instance can take more options. -### Backup strategy option +#### Backup strategy option > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8728) in GitLab 8.17. @@ -214,7 +209,7 @@ sudo gitlab-backup create STRATEGY=copy NOTE: **Note** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. -### Backup filename +#### Backup filename CAUTION: **Warning:** If you use a custom backup filename, you will not be able to @@ -231,7 +226,7 @@ For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. The resulting file will then be `dump_gitlab_backup.tar`. This is useful for systems that make use of rsync and incremental backups, and will result in considerably faster transfer speeds. -### Rsyncable +#### Rsyncable To make sure the generated archive is intelligently transferable by rsync, the `GZIP_RSYNCABLE=yes` option can be set. This will set the `--rsyncable` option to `gzip`. This is only useful in combination with setting [the Backup filename option](#backup-filename). @@ -244,7 +239,7 @@ sudo gitlab-backup create BACKUP=dump GZIP_RSYNCABLE=yes NOTE: **Note** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. -### Excluding specific directories from the backup +#### Excluding specific directories from the backup You can choose what should be exempt from the backup up by adding the environment variable `SKIP`. The available options are: @@ -278,7 +273,7 @@ For installations from source: sudo -u git -H bundle exec rake gitlab:backup:create SKIP=db,uploads RAILS_ENV=production ``` -### Skipping tar creation +#### Skipping tar creation The last part of creating a backup is generation of a `.tar` file containing all the parts. In some cases (for example, if the backup is picked up by other @@ -303,19 +298,19 @@ For installations from source: sudo -u git -H bundle exec rake gitlab:backup:create SKIP=tar RAILS_ENV=production ``` -### Uploading backups to a remote (cloud) storage +#### Uploading backups to a remote (cloud) storage -Starting with GitLab 7.4 you can let the backup script upload the '.tar' file it creates. +Starting with GitLab 7.4 you can let the backup script upload the `.tar` file it creates. It uses the [Fog library](http://fog.io/) to perform the upload. In the example below we use Amazon S3 for storage, but Fog also lets you use [other storage providers](http://fog.io/storage/). GitLab [imports cloud drivers](https://gitlab.com/gitlab-org/gitlab/blob/30f5b9a5b711b46f1065baf755e413ceced5646b/Gemfile#L88) -for AWS, Google, OpenStack Swift, Rackspace and Aliyun as well. A local driver is +for AWS, Google, OpenStack Swift, Rackspace, and Aliyun as well. A local driver is [also available](#uploading-to-locally-mounted-shares). [Read more about using object storage with GitLab](../administration/object_storage.md). -#### Using Amazon S3 +##### Using Amazon S3 For Omnibus GitLab packages: @@ -333,9 +328,9 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_remote_directory'] = 'my.s3.bucket' ``` -1. [Reconfigure GitLab] for the changes to take effect +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect -#### Digital Ocean Spaces +##### Digital Ocean Spaces This example can be used for a bucket in Amsterdam (AMS3). @@ -352,7 +347,7 @@ This example can be used for a bucket in Amsterdam (AMS3). gitlab_rails['backup_upload_remote_directory'] = 'my.s3.bucket' ``` -1. [Reconfigure GitLab] for the changes to take effect +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect NOTE: **Note:** If you see `400 Bad Request` by using Digital Ocean Spaces, the cause may be the @@ -360,7 +355,7 @@ usage of backup encryption. Remove or comment the line that contains `gitlab_rails['backup_encryption']` since Digital Ocean Spaces doesn't support encryption. -#### Other S3 Providers +##### Other S3 Providers Not all S3 providers are fully-compatible with the Fog library. For example, if you see `411 Length Required` errors after attempting to upload, you may @@ -397,7 +392,7 @@ For installations from source: # storage_class: 'STANDARD' ``` -1. [Restart GitLab] for the changes to take effect +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect If you are uploading your backups to S3 you will probably want to create a new IAM user with restricted access rights. To give the upload user access only for @@ -450,7 +445,7 @@ with the name of your bucket: } ``` -#### Using Google Cloud Storage +##### Using Google Cloud Storage If you want to use Google Cloud Storage to save backups, you'll have to create an access key from the Google console first: @@ -476,7 +471,7 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_remote_directory'] = 'my.google.bucket' ``` -1. [Reconfigure GitLab] for the changes to take effect +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect For installations from source: @@ -492,9 +487,9 @@ For installations from source: remote_directory: 'my.google.bucket' ``` -1. [Restart GitLab] for the changes to take effect +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect -#### Specifying a custom directory for backups +##### Specifying a custom directory for backups Note: This option only works for remote storage. If you want to group your backups you can pass a `DIRECTORY` environment variable: @@ -507,9 +502,9 @@ sudo gitlab-backup create DIRECTORY=weekly NOTE: **Note** For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. -### Uploading to locally mounted shares +#### Uploading to locally mounted shares -You may also send backups to a mounted share (`NFS` / `CIFS` / `SMB` / etc.) by +You may also send backups to a mounted share (for example, `NFS`,`CIFS`, or `SMB`) by using the Fog [`Local`](https://github.com/fog/fog-local#usage) storage provider. The directory pointed to by the `local_root` key **must** be owned by the `git` user **when mounted** (mounting with the `uid=` of the `git` user for `CIFS` and @@ -539,7 +534,7 @@ For Omnibus GitLab packages: gitlab_rails['backup_upload_remote_directory'] = 'gitlab_backups' ``` -1. [Reconfigure GitLab] for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. For installations from source: @@ -557,9 +552,9 @@ For installations from source: remote_directory: 'gitlab_backups' ``` -1. [Restart GitLab] for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. -### Backup archive permissions +#### Backup archive permissions The backup archives created by GitLab (`1393513186_2014_02_27_gitlab_backup.tar`) will have owner/group `git`/`git` and 0600 permissions by default. @@ -574,7 +569,7 @@ For Omnibus GitLab packages: gitlab_rails['backup_archive_permissions'] = 0644 # Makes the backup archives world-readable ``` -1. [Reconfigure GitLab] for the changes to take effect. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. For installations from source: @@ -585,9 +580,9 @@ For installations from source: archive_permissions: 0644 # Makes the backup archives world-readable ``` -1. [Restart GitLab] for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. -### Configuring cron to make daily backups +#### Configuring cron to make daily backups CAUTION: **Warning:** The following cron jobs do not [backup your GitLab configuration files](#storing-configuration-files) @@ -669,9 +664,9 @@ For installations from source: keep_time: 604800 ``` -1. [Restart GitLab] for the changes to take effect. +1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect. -## Restore +## Restore GitLab GitLab provides a simple command line interface to restore your whole installation, and is flexible enough to fit your needs. @@ -692,7 +687,7 @@ before restoring the backup. You need to have a working GitLab installation before you can perform a restore. This is mainly because the system user performing the restore actions (`git`) is usually not allowed to create or delete -the SQL database it needs to import data into ('gitlabhq_production'). +the SQL database it needs to import data into (`gitlabhq_production`). All existing data will be either erased (SQL) or moved to a separate directory (repositories, uploads). @@ -718,7 +713,7 @@ more of the following options: Read what the [backup timestamp is about](#backup-timestamp). - `force=yes` - Does not ask if the authorized_keys file should get regenerated and assumes 'yes' for warning that database tables will be removed, enabling the "Write to authorized_keys file" setting, and updating LDAP providers. -If you are restoring into directories that are mountpoints you will need to make +If you are restoring into directories that are mount points, you will need to make sure these directories are empty before attempting a restore. Otherwise GitLab will attempt to move these directories before restoring the new data and this would cause an error. @@ -902,6 +897,8 @@ will have all your repositories, but not any other data. ## Troubleshooting +The following are possible problems you might encounter with possible solutions. + ### Restoring database backup using Omnibus packages outputs warnings If you are using backup restore procedures you might encounter the following warnings: @@ -1079,12 +1076,9 @@ If you have changed the default filesystem location for the registry, you will want to run the `chown` against your custom location instead of `/var/opt/gitlab/gitlab-rails/shared/registry/docker`. -[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure -[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source - ### Backup fails to complete with Gzip error -While running the backup, you may receive a gzip error: +While running the backup, you may receive a Gzip error: ```shell sudo /opt/gitlab/bin/gitlab-backup create @@ -1098,5 +1092,5 @@ Backup failed If this happens, check the following: -1. Confirm there is sufficient disk space for the gzip operation. +1. Confirm there is sufficient disk space for the Gzip operation. 1. If NFS is being used, check if the mount option `timeout` is set. The default is `600`, and changing this to smaller values have resulted in this error. diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index ef69b1cce15..7908af8da84 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -1,4 +1,4 @@ -# Clean up +# Clean up **(CORE ONLY)** GitLab provides Rake tasks for cleaning up GitLab instances. @@ -7,9 +7,11 @@ GitLab provides Rake tasks for cleaning up GitLab instances. > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36628) in GitLab 12.10. DANGER: **Danger:** -Do not run this within 12 hours of a GitLab upgrade. This is to ensure that all background migrations have finished, which otherwise may lead to data loss. +Do not run this within 12 hours of a GitLab upgrade. This is to ensure that all background migrations +have finished, which otherwise may lead to data loss. -When you remove LFS files from a repository's history, they become orphaned and continue to consume disk space. With this Rake task, you can remove invalid references from the database, which +When you remove LFS files from a repository's history, they become orphaned and continue to consume +disk space. With this Rake task, you can remove invalid references from the database, which will allow garbage collection of LFS files. For example: @@ -177,3 +179,8 @@ sudo gitlab-rake gitlab:cleanup:sessions:active_sessions_lookup_keys # installation from source bundle exec rake gitlab:cleanup:sessions:active_sessions_lookup_keys RAILS_ENV=production ``` + +## Container Registry garbage collection + +Container Registry can use considerable amounts of disk space. To clear up +unused layers, the registry includes a [garbage collect command](../administration/packages/container_registry.md#container-registry-garbage-collection). diff --git a/doc/raketasks/features.md b/doc/raketasks/features.md index 5425a0a0667..1d68671a545 100644 --- a/doc/raketasks/features.md +++ b/doc/raketasks/features.md @@ -1,19 +1,21 @@ -# Namespaces +# Namespaces **(CORE ONLY)** + +This Rake task enables [namespaces](../user/group/index.md#namespaces) for projects. ## Enable usernames and namespaces for user projects -This command will enable the namespaces feature introduced in v4.0. It will move every project in its namespace folder. +This command will enable the namespaces feature introduced in GitLab 4.0. It will move every project in its namespace folder. Note: -- Because the **repository location will change**, you will need to **update all your Git URLs** to point to the new location. -- Username can be changed at **Profile ➔ Account**. - -**Example:** +- The **repository location will change**, so you will need to **update all your Git URLs** to + point to the new location. +- The username can be changed at **Profile > Account**. -Old path: `git@example.org:myrepo.git` +For example: -New path: `git@example.org:username/myrepo.git` or `git@example.org:groupname/myrepo.git` +- Old path: `git@example.org:myrepo.git`. +- New path: `git@example.org:username/myrepo.git` or `git@example.org:groupname/myrepo.git`. ```shell bundle exec rake gitlab:enable_namespaces RAILS_ENV=production diff --git a/doc/raketasks/generate_sample_prometheus_data.md b/doc/raketasks/generate_sample_prometheus_data.md index 78e8ef188bf..902be6862c8 100644 --- a/doc/raketasks/generate_sample_prometheus_data.md +++ b/doc/raketasks/generate_sample_prometheus_data.md @@ -1,15 +1,25 @@ -# Generate Sample Prometheus Data +# Generate sample Prometheus data **(CORE ONLY)** This command will run Prometheus queries for each of the metrics of a specific environment -for a series of time intervals: 30 minutes, 3 hours, 8 hours, 24 hours, 72 hours, and 7 days -to now. The results of each of query are stored under a `sample_metrics` directory as a yaml +for a series of time intervals to now: + +- 30 minutes +- 3 hours +- 8 hours +- 24 hours +- 72 hours +- 7 days + +The results of each of query are stored under a `sample_metrics` directory as a YAML file named by the metric's `identifier`. When the environmental variable `USE_SAMPLE_METRICS` is set, the Prometheus API query is re-routed to `Projects::Environments::SampleMetricsController` which loads the appropriate data set if it is present within the `sample_metrics` directory. -- This command requires an id from an Environment with an available Prometheus installation. +This command requires an ID from an environment with an available Prometheus installation. + +## Example -**Example:** +The following example demonstrates how to run the Rake task: ```shell bundle exec rake gitlab:generate_sample_prometheus_data[21] diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index cda742b6077..5229ce2ab08 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -1,61 +1,63 @@ -# Import bare repositories into your GitLab instance +# Import bare repositories **(CORE ONLY)** -## Notes +Rake tasks are available to import bare repositories into a GitLab instance. -- The owner of the project will be the first admin -- The groups will be created as needed, including subgroups -- The owner of the group will be the first admin -- Existing projects will be skipped -- Projects in hashed storage may be skipped (see [Importing bare repositories from hashed storage](#importing-bare-repositories-from-hashed-storage)) -- The existing Git repos will be moved from disk (removed from the original path) +Note that: -## How to use +- The owner of the project will be the first administrator. +- The groups will be created as needed, including subgroups. +- The owner of the group will be the first administrator. +- Existing projects will be skipped. +- Projects in hashed storage may be skipped. For more information, see + [Importing bare repositories from hashed storage](#importing-bare-repositories-from-hashed-storage). +- The existing Git repositories will be moved from disk (removed from the original path). -### Create a new folder to import your Git repositories from +To import bare repositories into a GitLab instance: -The new folder needs to have Git user ownership and read/write/execute access for Git user and its group: +1. Create a new folder to import your Git repositories from. The new folder needs to have Git user + ownership and read/write/execute access for Git user and its group: -```shell -sudo -u git mkdir -p /var/opt/gitlab/git-data/repository-import-<date>/new_group -``` - -### Copy your bare repositories inside this newly created folder + ```shell + sudo -u git mkdir -p /var/opt/gitlab/git-data/repository-import-<date>/new_group + ``` -- Any `.git` repositories found on any of the subfolders will be imported as projects -- Groups will be created as needed, these could be nested folders. Example: +1. Copy your bare repositories inside this newly created folder. Note: -If we copy the repos to `/var/opt/gitlab/git-data/repository-import-<date>`, and repo A needs to be under the groups G1 and G2, it will -have to be created under those folders: `/var/opt/gitlab/git-data/repository-import-<date>/G1/G2/A.git`. + - Any `.git` repositories found on any of the subfolders will be imported as projects. + - Groups will be created as needed, these could be nested folders. -```shell -sudo cp -r /old/git/foo.git /var/opt/gitlab/git-data/repository-import-<date>/new_group/ + For example, if we copy the repositories to `/var/opt/gitlab/git-data/repository-import-<date>`, + and repository `A` needs to be under the groups `G1` and `G2`, it must be created under those folders: + `/var/opt/gitlab/git-data/repository-import-<date>/G1/G2/A.git`. -# Do this once when you are done copying git repositories -sudo chown -R git:git /var/opt/gitlab/git-data/repository-import-<date> -``` + ```shell + sudo cp -r /old/git/foo.git /var/opt/gitlab/git-data/repository-import-<date>/new_group/ -`foo.git` needs to be owned by the `git` user and `git` users group. + # Do this once when you are done copying git repositories + sudo chown -R git:git /var/opt/gitlab/git-data/repository-import-<date> + ``` -If you are using an installation from source, replace `/var/opt/gitlab/` with `/home/git`. + `foo.git` needs to be owned by the `git` user and `git` users group. -### Run the command below depending on your type of installation + If you are using an installation from source, replace `/var/opt/gitlab/` with `/home/git`. -#### Omnibus Installation +1. Run the following command depending on your type of installation: -```shell -sudo gitlab-rake gitlab:import:repos['/var/opt/gitlab/git-data/repository-import-<date>'] -``` + - Omnibus Installation -#### Installation from source + ```shell + sudo gitlab-rake gitlab:import:repos['/var/opt/gitlab/git-data/repository-import-<date>'] + ``` -Before running this command you need to change the directory to where your GitLab installation is located: + - Installation from source. Before running this command you need to change to the directory where + your GitLab installation is located: -```shell -cd /home/git/gitlab -sudo -u git -H bundle exec rake gitlab:import:repos['/var/opt/gitlab/git-data/repository-import-<date>'] RAILS_ENV=production -``` + ```shell + cd /home/git/gitlab + sudo -u git -H bundle exec rake gitlab:import:repos['/var/opt/gitlab/git-data/repository-import-<date>'] RAILS_ENV=production + ``` -#### Example output +## Example output ```plaintext Processing /var/opt/gitlab/git-data/repository-import-1/a/b/c/blah.git @@ -73,8 +75,6 @@ Processing /var/opt/gitlab/git-data/repository-import-1/group/xyz.git ## Importing bare repositories from hashed storage -### Background - Projects in legacy storage have a directory structure that mirrors their full project path in GitLab, including their namespace structure. This information is leveraged by the bare repository importer to import projects into their proper @@ -86,17 +86,17 @@ improved performance and data integrity. See [Repository Storage Types](../administration/repository_storage_types.md) for more details. -### Which repositories are importable? +The repositories that are importable depends on the version of GitLab. -#### GitLab 10.3 or earlier +### GitLab 10.3 or earlier Importing bare repositories from hashed storage is unsupported. -#### GitLab 10.4 and later +### GitLab 10.4 and later To support importing bare repositories from hashed storage, GitLab 10.4 and later stores the full project path with each repository, in a special section of -the Git repository's config file. This section is formatted as follows: +the Git repository's configuration file. This section is formatted as follows: ```ini [gitlab] diff --git a/doc/raketasks/list_repos.md b/doc/raketasks/list_repos.md index 10e6cb04bfa..411de52e379 100644 --- a/doc/raketasks/list_repos.md +++ b/doc/raketasks/list_repos.md @@ -1,7 +1,8 @@ -# Listing repository directories +# Listing repository directories **(CORE ONLY)** -You can print a list of all Git repositories on disk managed by -GitLab with the following command: +You can print a list of all Git repositories on disk managed by GitLab. + +To print a list, run the following command: ```shell # Omnibus @@ -12,10 +13,13 @@ cd /home/git/gitlab sudo -u git -H bundle exec rake gitlab:list_repos RAILS_ENV=production ``` -If you only want to list projects with recent activity you can pass -a date with the 'SINCE' environment variable. The time you specify -is parsed by the Rails [TimeZone#parse -function](https://api.rubyonrails.org/classes/ActiveSupport/TimeZone.html#method-i-parse). +NOTE: **Note:** +The results use the default ordering of the GitLab Rails application. + +## Limit search results + +To list only projects with recent activity, pass a date with the `SINCE` environment variable. The +time you specify is parsed by the Rails [TimeZone#parse function](https://api.rubyonrails.org/classes/ActiveSupport/TimeZone.html#method-i-parse). ```shell # Omnibus @@ -25,6 +29,3 @@ sudo gitlab-rake gitlab:list_repos SINCE='Sep 1 2015' cd /home/git/gitlab sudo -u git -H bundle exec rake gitlab:list_repos RAILS_ENV=production SINCE='Sep 1 2015' ``` - -Note that the projects listed are NOT sorted by activity; they use -the default ordering of the GitLab Rails application. diff --git a/doc/raketasks/migrate_snippets.md b/doc/raketasks/migrate_snippets.md new file mode 100644 index 00000000000..fce91ab703e --- /dev/null +++ b/doc/raketasks/migrate_snippets.md @@ -0,0 +1,96 @@ +# Migration to Versioned Snippets **(CORE ONLY)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215861) in GitLab 13.0. + +In GitLab 13.0, [GitLab Snippets are backed by Git repositories](../user/snippets.md#versioned-snippets). +This means that the snippet content will be stored in the repository +and users can update it directly through Git. + +Nevertheless, existing GitLab Snippets have to be migrated to this new functionality. +For each snippet, a new repository is created and the snippet content is committed +to the repository inside a file whose name is the file name used in the snippet +as well. + +GitLab performs this migration through a [Background Migration](../development/background_migrations.md) +automatically when the GitLab instance is upgrade to 13.0 or a higher version. +However, if the migration fails for any of the snippets, they still need +to be migrated individually. + +The following Rake tasks will help with that process. + +## Migrate specific snippets to Git + +In case you want to migrate a range of snippets, run the tasks as described below. + +For Omnibus installations, run: + +```shell +sudo gitlab-rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 +``` + +For installations from source code, run: + +```shell +bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 +``` + +There is a default limit (100) to the number of ids supported in the migration +process. You can modify this limit by using the env variable `LIMIT`. + +```shell +sudo gitlab-rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 LIMIT=50 +``` + +For installations from source code, run: + +```shell +bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 LIMIT=50 +``` + +## Show whether the snippet background migration is running + +In case you want to check the status of the snippet background migration, +whether it is running or not, you can use the following task. + +For Omnibus installations, run: + +```shell +sudo gitlab-rake gitlab:snippets:migration_status +``` + +For installations from source code, run: + +```shell +bundle exec rake gitlab:snippets:migration_status RAILS_ENV=production +``` + +## List non-migrated snippets + +With the following task, you can get the ids of all of the snippets +that haven't been migrated yet or failed to migrate. + +For Omnibus installations, run: + +```shell +sudo gitlab-rake gitlab:snippets:list_non_migrated +``` + +For installations from source code, run: + +```shell +bundle exec rake gitlab:snippets:list_non_migrated RAILS_ENV=production +``` + +As the number of non-migrated snippets can be large, we limit +by default the size of the number of ids returned to 100. You can +modify this limit by using the env variable `LIMIT`. + +```shell +sudo gitlab-rake gitlab:snippets:list_non_migrated LIMIT=200 +``` + +For installations from source code, run: + +```shell +bundle exec rake gitlab:snippets:list_non_migrated RAILS_ENV=production LIMIT=200 +``` diff --git a/doc/raketasks/user_management.md b/doc/raketasks/user_management.md index ffb1baa0b6e..61e30f9ddb1 100644 --- a/doc/raketasks/user_management.md +++ b/doc/raketasks/user_management.md @@ -1,7 +1,11 @@ -# User management +# User management **(CORE ONLY)** + +GitLab provides Rake tasks for user management. ## Add user as a developer to all projects +To add a user as a developer to all projects, run: + ```shell # omnibus-gitlab sudo gitlab-rake gitlab:import:user_to_projects[username@domain.tld] @@ -12,9 +16,7 @@ bundle exec rake gitlab:import:user_to_projects[username@domain.tld] RAILS_ENV=p ## Add all users to all projects -Notes: - -- admin users are added as maintainers +To add all users to all projects, run: ```shell # omnibus-gitlab @@ -24,8 +26,13 @@ sudo gitlab-rake gitlab:import:all_users_to_all_projects bundle exec rake gitlab:import:all_users_to_all_projects RAILS_ENV=production ``` +NOTE: **Note:** +Admin users are added as maintainers. + ## Add user as a developer to all groups +To add a user as a developer to all groups, run: + ```shell # omnibus-gitlab sudo gitlab-rake gitlab:import:user_to_groups[username@domain.tld] @@ -36,9 +43,7 @@ bundle exec rake gitlab:import:user_to_groups[username@domain.tld] RAILS_ENV=pro ## Add all users to all groups -Notes: - -- admin users are added as owners so they can add additional users to the group +To add all users to all groups, run: ```shell # omnibus-gitlab @@ -48,19 +53,25 @@ sudo gitlab-rake gitlab:import:all_users_to_all_groups bundle exec rake gitlab:import:all_users_to_all_groups RAILS_ENV=production ``` -## Maintain tight control over the number of active users on your GitLab installation +NOTE: **Note:** +Admin users are added as owners so they can add additional users to the group. + +## Control the number of active users -- Enable this setting to keep new users blocked until they have been cleared by the admin (default: false). +Enable this setting to keep new users blocked until they have been cleared by the administrator. +Defaults to `false`: ```plaintext block_auto_created_users: false ``` -## Disable Two-factor Authentication (2FA) for all users +## Disable two-factor authentication for all users -This task will disable 2FA for all users that have it enabled. This can be +This task disables two-factor authentication (2FA) for all users that have it enabled. This can be useful if GitLab's `config/secrets.yml` file has been lost and users are unable -to login, for example. +to log in, for example. + +To disable two-factor authentication for all users, run: ```shell # omnibus-gitlab @@ -70,65 +81,65 @@ sudo gitlab-rake gitlab:two_factor:disable_for_all_users bundle exec rake gitlab:two_factor:disable_for_all_users RAILS_ENV=production ``` -## Rotate Two-factor Authentication (2FA) encryption key +## Rotate two-factor authentication encryption key -GitLab stores the secret data enabling 2FA to work in an encrypted database -column. The encryption key for this data is known as `otp_key_base`, and is +GitLab stores the secret data required for two-factor authentication (2FA) in an encrypted +database column. The encryption key for this data is known as `otp_key_base`, and is stored in `config/secrets.yml`. If that file is leaked, but the individual 2FA secrets have not, it's possible to re-encrypt those secrets with a new encryption key. This allows you to change the leaked key without forcing all users to change their 2FA details. -First, look up the old key. This is in the `config/secrets.yml` file, but -**make sure you're working with the production section**. The line you're -interested in will look like this: +To rotate the two-factor authentication encryption key: -```yaml -production: - otp_key_base: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff -``` +1. Look up the old key. This is in the `config/secrets.yml` file, but **make sure you're working + with the production section**. The line you're interested in will look like this: -Next, generate a new secret: + ```yaml + production: + otp_key_base: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + ``` -```shell -# omnibus-gitlab -sudo gitlab-rake secret +1. Generate a new secret: -# installation from source -bundle exec rake secret RAILS_ENV=production -``` + ```shell + # omnibus-gitlab + sudo gitlab-rake secret -Now you need to stop the GitLab server, back up the existing secrets file and -update the database: + # installation from source + bundle exec rake secret RAILS_ENV=production + ``` -```shell -# omnibus-gitlab -sudo gitlab-ctl stop -sudo cp config/secrets.yml config/secrets.yml.bak -sudo gitlab-rake gitlab:two_factor:rotate_key:apply filename=backup.csv old_key=<old key> new_key=<new key> +1. Stop the GitLab server, back up the existing secrets file, and update the database: -# installation from source -sudo /etc/init.d/gitlab stop -cp config/secrets.yml config/secrets.yml.bak -bundle exec rake gitlab:two_factor:rotate_key:apply filename=backup.csv old_key=<old key> new_key=<new key> RAILS_ENV=production -``` + ```shell + # omnibus-gitlab + sudo gitlab-ctl stop + sudo cp config/secrets.yml config/secrets.yml.bak + sudo gitlab-rake gitlab:two_factor:rotate_key:apply filename=backup.csv old_key=<old key> new_key=<new key> -The `<old key>` value can be read from `config/secrets.yml`; `<new key>` was -generated earlier. The **encrypted** values for the user 2FA secrets will be -written to the specified `filename` - you can use this to rollback in case of -error. + # installation from source + sudo /etc/init.d/gitlab stop + cp config/secrets.yml config/secrets.yml.bak + bundle exec rake gitlab:two_factor:rotate_key:apply filename=backup.csv old_key=<old key> new_key=<new key> RAILS_ENV=production + ``` -Finally, change `config/secrets.yml` to set `otp_key_base` to `<new key>` and -restart. Again, make sure you're operating in the **production** section. + The `<old key>` value can be read from `config/secrets.yml` (`<new key>` was + generated earlier). The **encrypted** values for the user 2FA secrets will be + written to the specified `filename`. You can use this to rollback in case of + error. -```shell -# omnibus-gitlab -sudo gitlab-ctl start +1. Change `config/secrets.yml` to set `otp_key_base` to `<new key>` and restart. Again, make sure + you're operating in the **production** section. -# installation from source -sudo /etc/init.d/gitlab start -``` + ```shell + # omnibus-gitlab + sudo gitlab-ctl start + + # installation from source + sudo /etc/init.d/gitlab start + ``` If there are any problems (perhaps using the wrong value for `old_key`), you can restore your backup of `config/secrets.yml` and rollback the changes: diff --git a/doc/raketasks/web_hooks.md b/doc/raketasks/web_hooks.md index 22084c862ba..7bd2ed311d2 100644 --- a/doc/raketasks/web_hooks.md +++ b/doc/raketasks/web_hooks.md @@ -1,60 +1,78 @@ # Webhooks administration **(CORE ONLY)** -## Add a webhook for **ALL** projects +GitLab provides Rake tasks for webhooks management. + +Requests to the [local network by webhooks](../security/webhooks.md) can be allowed or blocked by an +administrator. + +## Add a webhook to all projects + +To add a webhook to all projects, run: ```shell # omnibus-gitlab sudo gitlab-rake gitlab:web_hook:add URL="http://example.com/hook" + # source installations bundle exec rake gitlab:web_hook:add URL="http://example.com/hook" RAILS_ENV=production ``` -## Add a webhook for projects in a given **NAMESPACE** +## Add a webhook to projects in a namespace + +To add a webhook to all projects in a specific namespace, run: ```shell # omnibus-gitlab -sudo gitlab-rake gitlab:web_hook:add URL="http://example.com/hook" NAMESPACE=acme +sudo gitlab-rake gitlab:web_hook:add URL="http://example.com/hook" NAMESPACE=<namespace> + # source installations -bundle exec rake gitlab:web_hook:add URL="http://example.com/hook" NAMESPACE=acme RAILS_ENV=production +bundle exec rake gitlab:web_hook:add URL="http://example.com/hook" NAMESPACE=<namespace> RAILS_ENV=production ``` -## Remove a webhook from **ALL** projects using +## Remove a webhook from projects + +To remove a webhook from all projects, run: ```shell # omnibus-gitlab sudo gitlab-rake gitlab:web_hook:rm URL="http://example.com/hook" + # source installations bundle exec rake gitlab:web_hook:rm URL="http://example.com/hook" RAILS_ENV=production ``` -## Remove a webhook from projects in a given **NAMESPACE** +## Remove a webhook from projects in a namespace + +To remove a webhook from projects in a specific namespace, run: ```shell # omnibus-gitlab -sudo gitlab-rake gitlab:web_hook:rm URL="http://example.com/hook" NAMESPACE=acme +sudo gitlab-rake gitlab:web_hook:rm URL="http://example.com/hook" NAMESPACE=<namespace> + # source installations -bundle exec rake gitlab:web_hook:rm URL="http://example.com/hook" NAMESPACE=acme RAILS_ENV=production +bundle exec rake gitlab:web_hook:rm URL="http://example.com/hook" NAMESPACE=<namespace> RAILS_ENV=production ``` -## List **ALL** webhooks +## List all webhooks + +To list all webhooks, run: ```shell # omnibus-gitlab sudo gitlab-rake gitlab:web_hook:list + # source installations bundle exec rake gitlab:web_hook:list RAILS_ENV=production ``` -## List the webhooks from projects in a given **NAMESPACE** +## List webhooks for projects in a namespace + +To list all webhook for projects in a specified namespace, run: ```shell # omnibus-gitlab -sudo gitlab-rake gitlab:web_hook:list NAMESPACE=acme +sudo gitlab-rake gitlab:web_hook:list NAMESPACE=<namespace> + # source installations -bundle exec rake gitlab:web_hook:list NAMESPACE=acme RAILS_ENV=production +bundle exec rake gitlab:web_hook:list NAMESPACE=<namespace> RAILS_ENV=production ``` - -## Local requests in webhooks - -[Requests to local network by webhooks](../security/webhooks.md) can be allowed -or blocked by an administrator. diff --git a/doc/raketasks/x509_signatures.md b/doc/raketasks/x509_signatures.md index dd518997f7b..f7c47794690 100644 --- a/doc/raketasks/x509_signatures.md +++ b/doc/raketasks/x509_signatures.md @@ -1,21 +1,24 @@ -# X509 signatures +# X.509 signatures **(CORE ONLY)** -When [signing commits with x509](../user/project/repository/x509_signed_commits/index.md) -the trust anchor might change and the signatures stored within the database have -to be updated. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/122159) in GitLab 12.10. -## Update all x509 signatures +When [signing commits with X.509](../user/project/repository/x509_signed_commits/index.md), +the trust anchor might change and the signatures stored within the database must be updated. -This task loops through all X509 signed commits and updates their verification -based on current certificate store. +## Update all X.509 signatures -**Omnibus Installation** +This task loops through all X.509 signed commits and updates their verification based on current +certificate store. + +To update all X.509 signatures, run: + +**Omnibus Installations:** ```shell sudo gitlab-rake gitlab:x509:update_signatures ``` -**Source Installation** +**Source Installations:** ```shell sudo -u git -H bundle exec rake gitlab:x509:update_signatures RAILS_ENV=production diff --git a/doc/security/README.md b/doc/security/README.md index c21d99658b8..e2375c0f0b5 100644 --- a/doc/security/README.md +++ b/doc/security/README.md @@ -23,6 +23,4 @@ type: index ## Securing your GitLab installation -To make sure your GitLab instance is safe and secure, please consider implementing -[Sign up restrictions](../user/admin_area/settings/sign_up_restrictions.md) to avoid -malicious users creating accounts. +Consider access control features like [Sign up restrictions](../user/admin_area/settings/sign_up_restrictions.md) and [Authentication options](../topics/authentication/) to harden your GitLab instance and minimize the risk of unwanted user account creation. diff --git a/doc/security/asset_proxy.md b/doc/security/asset_proxy.md index d042fb3efe7..91a35c2f2a9 100644 --- a/doc/security/asset_proxy.md +++ b/doc/security/asset_proxy.md @@ -63,6 +63,6 @@ For example, the following is a link to an image in Markdown: The following is an example of a source link that could result: -```text +```plaintext http://proxy.gitlab.example.com/f9dd2b40157757eb82afeedbf1290ffb67a3aeeb/68747470733a2f2f61626f75742e6769746c61622e636f6d2f696d616765732f70726573732f6c6f676f2f6a70672f6769746c61622d69636f6e2d7267622e6a7067 ``` diff --git a/doc/security/crime_vulnerability.md b/doc/security/crime_vulnerability.md index 93edbc69eb0..2496029d93e 100644 --- a/doc/security/crime_vulnerability.md +++ b/doc/security/crime_vulnerability.md @@ -14,14 +14,14 @@ authenticated web session, allowing the launching of further attacks. The TLS Protocol CRIME Vulnerability affects systems that use data compression over HTTPS. Your system might be vulnerable to the CRIME vulnerability if you use -SSL Compression (for example, gzip) or SPDY (which optionally uses compression). +SSL Compression (for example, Gzip) or SPDY (which optionally uses compression). -GitLab supports both gzip and [SPDY][ngx-spdy] and mitigates the CRIME -vulnerability by deactivating gzip when HTTPS is enabled. The sources of the +GitLab supports both Gzip and [SPDY](http://nginx.org/en/docs/http/ngx_http_spdy_module.html) and mitigates the CRIME +vulnerability by deactivating Gzip when HTTPS is enabled. The sources of the files are here: -- [Source installation NGINX file][source-nginx] -- [Omnibus installation NGINX file][omnibus-nginx] +- [Source installation NGINX file](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/nginx/gitlab-ssl) +- [Omnibus installation NGINX file](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb) Although SPDY is enabled in Omnibus installations, CRIME relies on compression (the 'C') and the default compression level in NGINX's SPDY module is 0 @@ -29,7 +29,7 @@ Although SPDY is enabled in Omnibus installations, CRIME relies on compression ## Nessus -The Nessus scanner, [reports a possible CRIME vulnerability][nessus] in GitLab +The Nessus scanner, [reports a possible CRIME vulnerability](https://www.tenable.com/plugins/index.php?view=single&id=62565) in GitLab similar to the following format: ```plaintext @@ -55,15 +55,9 @@ vulnerability. ## References -- NGINX ["Module ngx_http_spdy_module"][ngx-spdy] -- Tenable Network Security, Inc. ["Transport Layer Security (TLS) Protocol CRIME Vulnerability"][nessus] -- Wikipedia contributors, ["CRIME"][wiki-crime] Wikipedia, The Free Encyclopedia - -[source-nginx]: https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/nginx/gitlab-ssl -[omnibus-nginx]: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb -[ngx-spdy]: http://nginx.org/en/docs/http/ngx_http_spdy_module.html -[nessus]: https://www.tenable.com/plugins/index.php?view=single&id=62565 -[wiki-crime]: https://en.wikipedia.org/wiki/CRIME +- NGINX ["Module ngx_http_spdy_module"](http://nginx.org/en/docs/http/ngx_http_spdy_module.html) +- Tenable Network Security, Inc. ["Transport Layer Security (TLS) Protocol CRIME Vulnerability"](https://www.tenable.com/plugins/index.php?view=single&id=62565) +- Wikipedia contributors, ["CRIME"](https://en.wikipedia.org/wiki/CRIME) Wikipedia, The Free Encyclopedia <!-- ## Troubleshooting diff --git a/doc/security/rack_attack.md b/doc/security/rack_attack.md index 9ce2a9bb1ae..5d18746e4e0 100644 --- a/doc/security/rack_attack.md +++ b/doc/security/rack_attack.md @@ -36,27 +36,20 @@ will be enabled: ### Protected paths throttle -NOTE: **Note:** Omnibus GitLab protected paths throttle is deprecated and is scheduled for removal in -GitLab 13.0. Please refer to [Migrate settings from GitLab 12.3 and earlier](../user/admin_area/settings/protected_paths.md#migrate-settings-from-gitlab-123-and-earlier). - GitLab responds with HTTP status code `429` to POST requests at protected paths that exceed 10 requests per minute per IP address. By default, protected paths are: -```ruby -default['gitlab']['gitlab-rails']['rack_attack_protected_paths'] = [ - '/users/password', - '/users/sign_in', - '/api/#{API::API.version}/session.json', - '/api/#{API::API.version}/session', - '/users', - '/users/confirmation', - '/unsubscribes/', - '/import/github/personal_access_token', - '/admin/session' -] -``` +- `/users/password` +- `/users/sign_in` +- `/api/#{API::API.version}/session.json` +- `/api/#{API::API.version}/session` +- `/users` +- `/users/confirmation` +- `/unsubscribes/` +- `/import/github/personal_access_token` +- `/admin/session` This header is included in responses to blocked requests: @@ -141,9 +134,6 @@ taken in order to enable protection for your GitLab instance: config.middleware.use Rack::Attack ``` -1. Copy `config/initializers/rack_attack.rb.example` to `config/initializers/rack_attack.rb` -1. Open `config/initializers/rack_attack.rb`, review the - `paths_to_be_protected`, and add any other path you need protecting 1. Restart GitLab: ```shell diff --git a/doc/security/webhooks.md b/doc/security/webhooks.md index bd05cbff05e..27f79dbdf66 100644 --- a/doc/security/webhooks.md +++ b/doc/security/webhooks.md @@ -78,7 +78,7 @@ will whitelist all ports on all IPs in that range. Example: -```text +```plaintext example.com;gitlab.example.com 127.0.0.1,1:0:0:0:0:0:0:1 127.0.0.0/8 1:0:0:0:0:0:0:0/124 diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index c81310edc44..3978506cbf9 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -75,9 +75,10 @@ count as active users in the subscription period in which they were originally a - Members with Guest permissions on an Ultimate subscription. - GitLab-created service accounts: `Ghost User` and `Support Bot`. -##### User Statistics +##### Users statistics -A breakdown of the users within your instance including active, billable and blocked can be found by navigating to **Admin Area > Overview > Dashboard** and selecting `Users Statistics` button within the `Users` widget.. +To view a breakdown of the users within your instance, including active, billable, and blocked, go to **Admin Area > Overview > Dashboard** and select **Users statistics** in the **Users** section. +For more details, see [Users statistics](../user/admin_area/index.md#users-statistics). NOTE: **Note:** If you have LDAP integration enabled, anyone in the configured domain can sign up for a GitLab account. This can result in an unexpected bill at time of renewal. Consider [disabling new signups](../user/admin_area/settings/sign_up_restrictions.md) and managing new users manually instead. @@ -126,6 +127,7 @@ instance, ensure you're purchasing enough seats to With the [Customers Portal](https://customers.gitlab.com/) you can: - [Change billing information](#change-billing-information) +- [Change the payment method](#change-payment-method) - [Change the linked account](#change-the-linked-account) - [Change the associated namespace](#change-the-associated-namespace) @@ -143,6 +145,15 @@ Future purchases will use the information in this section. The email listed in this section is used for the Customers Portal login and for license-related email communication. +### Change payment method + +To change payment method or update credit card information: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select the **My account** drop-down and click on **Payment methods**. +1. **Edit** the existing payment method information or **Add new payment method**. +1. Save changes. + ### Change the linked account To change the GitLab.com account associated with a Customers Portal @@ -171,6 +182,8 @@ Subscription charges are calculated based on the total number of users in a grou ## View your subscription +### View your GitLab.com subscription + To see the status of your GitLab.com subscription, log into GitLab.com and go to the **Billing** section of the relevant namespace: - For individuals: @@ -190,6 +203,13 @@ The following table describes details of your subscription for groups: | Subscription start date | Date your subscription started. If this is for a Free plan, is the date you transitioned off your group's paid plan. | | Subscription end date | Date your current subscription will end. Does not apply to Free plans. | +### View your self-managed subscription + +To view the status of your self-managed subscription, log into the self-managed instance and go to the **License** page. + + 1. Go to **{admin}** **Admin Area**. + 1. From the left-hand menu, select **License**. + ## Renew your subscription To renew your subscription, [prepare for renewal by reviewing your account](#prepare-for-renewal-by-reviewing-your-account), then do one of the following: diff --git a/doc/telemetry/backend.md b/doc/telemetry/backend.md deleted file mode 100644 index c571439af8a..00000000000 --- a/doc/telemetry/backend.md +++ /dev/null @@ -1,34 +0,0 @@ -# Backend tracking guide - -GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. - -## Tracking in Ruby - -Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: - -| argument | type | default value | description | -|:-----------|:-------|:---------------------------|:------------| -| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | -| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). These will be set as empty strings if you don't provide them. | - -Tracking can be viewed as either tracking user behavior, or can be utilized for instrumentation to monitor and visual performance over time in an area or aspect of code. - -For example: - -```ruby -class Projects::CreateService < BaseService - def execute - project = Project.create(params) - - Gitlab::Tracking.event('Projects::CreateService', 'create_project', - label: project.errors.full_messages.to_sentence, - value: project.valid? - ) - end -end -``` - -### Performance - -We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. diff --git a/doc/telemetry/frontend.md b/doc/telemetry/frontend.md deleted file mode 100644 index fcd394500ec..00000000000 --- a/doc/telemetry/frontend.md +++ /dev/null @@ -1,167 +0,0 @@ -# Frontend tracking guide - -GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to utilize tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). - -| field | type | default value | description | -|:-----------|:-------|:---------------------------|:------------| -| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | -| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | - -## Tracking in HAML (or Vue Templates) - -When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute will automatically have event tracking bound on clicks. - -Below is an example of `data-track-*` attributes assigned to a button: - -```haml -%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } -``` - -```html -<button class="btn" - data-track-event="click_button" - data-track-label="template_preview" - data-track-property="my-template" -/> -``` - -Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows for them to be properly handled on rerendering and changes to the DOM, but it's important to know that because of the way these events are bound, click events shouldn't be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you'll need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). - -Below is a list of supported `data-track-*` attributes: - -| attribute | required | description | -|:----------------------|:---------|:------------| -| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | -| `data-track-label` | false | The `label` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | -| `data-track-property` | false | The `property` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | -| `data-track-value` | false | The `value` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). If omitted, this will be the elements `value` property or an empty string. For checkboxes, the default value will be the element's checked attribute or `false` when unchecked. | -| `data-track-context` | false | The `context` as described [in our Feature Instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy). | - -## Tracking within Vue components - -There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. - -```javascript -import Tracking from '~/tracking'; -const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); -``` - -You can provide default options that will be passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. - -You can then use the mixin normally in your component with the `mixin`, Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These will override any defaults and allows the values to be dynamic from props, or based on state. - -```javascript -export default { - mixins: [trackingMixin], - // ...[component implementation]... - data() { - return { - expanded: false, - tracking: { - label: 'left_sidebar' - } - }; - }, -} -``` - -The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. - -```javascript -export default { - mixins: [Tracking.mixin({ label: 'right_sidebar' })], - data() { - return { - expanded: false, - }; - }, - methods: { - toggle() { - this.expanded = !this.expanded; - this.track('click_toggle', { value: this.expanded }) - } - } -}; -``` - -And if needed within the template, you can use the `track` method directly as well. - -```html -<template> - <div> - <a class="toggle" @click.prevent="toggle">Toggle</a> - <div v-if="expanded"> - <p>Hello world!</p> - <a @click.prevent="track('click_action')">Track an event</a> - </div> - </div> -</template> -``` - -## Tracking in raw JavaScript - -Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. - -```javascript -import Tracking from '~/tracking'; - -const button = document.getElementById('create_from_template_button'); -button.addEventListener('click', () => { - Tracking.event('dashboard:projects:index', 'click_button', { - label: 'create_from_template', - property: 'template_preview', - value: 'rails', - }); -}) -``` - -## Tests and test helpers - -In Jest particularly in vue tests, you can use the following: - -```javascript -import { mockTracking } from 'helpers/tracking_helper'; - -describe('MyTracking', () => { - let spy; - - beforeEach(() => { - spy = mockTracking('_category_', wrapper.element, jest.spyOn); - }); - - it('tracks an event when clicked on feedback', () => { - wrapper.find('.discover-feedback-icon').trigger('click'); - - expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { - label: 'security-discover-feedback-cta', - property: '0', - }); - }); -}); - -``` - -In obsolete Karma tests it's used as below: - -```javascript -import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; - -describe('my component', () => { - let trackingSpy; - - beforeEach(() => { - const vm = mountComponent(MyComponent); - trackingSpy = mockTracking('_category_', vm.$el, spyOn); - }); - - it('tracks an event when toggled', () => { - triggerEvent('a.toggle'); - - expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { - label: 'right_sidebar', - property: 'confidentiality', - }); - }); -}); -``` diff --git a/doc/telemetry/index.md b/doc/telemetry/index.md index 55a7fad86be..977b93b712e 100644 --- a/doc/telemetry/index.md +++ b/doc/telemetry/index.md @@ -1,72 +1,5 @@ -# Event tracking +--- +redirect_to: '../development/telemetry/index.md' +--- -At GitLab, we encourage event tracking so we can iterate on and improve the project and user experience. - -We do this by running experiments, and collecting analytics for features and feature variations. This is: - -- So we generally know engagement. -- A way to approach A/B testing. - -As developers, we should attempt to add tracking and instrumentation where possible. This enables the Product team to better understand: - -- User engagement. -- Usage patterns. -- Other metrics that can potentially be improved on. - -To maintain consistency and not adversely effect performance, we have some basic tracking functionality exposed at both the frontend and backend layers that can be utilized while building new features or updating existing features. - -We also encourage users to enable tracking, and we embrace full transparency with our tracking approach so it can be easily understood and trusted. By enabling tracking, users can: - -- Contribute back to the wider community. -- Help GitLab improve on the product. - -## Implementing tracking - -Event tracking can be implemented on either the frontend or the backend layers, and each can be approached slightly differently since they have slightly different concerns. - -In GitLab, many actions can be initiated via the web interface, but they can also be initiated via an API client (an iOS applications is a good example of this), or via `git` directly. Crucially, this means that tracking should be considered holistically for the feature that's being instrumented. - -The data team should be involved when defining analytics and can be consulted when coming up with ways of presenting data that's being tracked. This allows our event data to be considered carefully and presented in ways that may reveal details about user engagement that may not be fully understood or interactions where we can make improvements. You can [contact the data team](https://about.gitlab.com/handbook/business-ops/data-team/#contact-us) and consult with them when defining tracking strategies. - -### Frontend - -Generally speaking, the frontend can track user actions and events, like: - -- Clicking links or buttons. -- Submitting forms. -- Other typically interface-driven actions. - -See [Frontend tracking guide](frontend.md). - -### Backend - -From the backend, the events that are tracked will likely consist of things like the creation or deletion of records and other events that might be triggered from layers that aren't necessarily only available in the interface. - -See [Backend tracking guide](backend.md). - -Also, see [Instrumenting Ruby code](../development/instrumentation.md) if you are instrumenting application performance metrics for Ruby code. - -## Enabling tracking - -Tracking can be enabled at: - -- The instance level, which will enable tracking on both the frontend and backend layers. -- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser will also not be tracked from a user level. - -We utilize Snowplow for the majority of our tracking strategy, and it can be enabled by navigating to: - -- **Admin Area > Settings > Integrations** in the UI. -- `admin/application_settings/integrations` in your browser. - -The following configuration is required: - -| Name | Value | -| ------------- | ------------------------- | -| Collector | `snowplow.trx.gitlab.net` | -| Site ID | `gitlab` | -| Cookie domain | `.gitlab.com` | - -Once enabled, tracking events can be inspected locally by either: - -- Looking at the network panel of the browser's development tools -- Using the [Snowplow Chrome Extension](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm). +This document was moved to [another location](../development/telemetry/index.md). diff --git a/doc/telemetry/snowplow.md b/doc/telemetry/snowplow.md new file mode 100644 index 00000000000..977b93b712e --- /dev/null +++ b/doc/telemetry/snowplow.md @@ -0,0 +1,5 @@ +--- +redirect_to: '../development/telemetry/index.md' +--- + +This document was moved to [another location](../development/telemetry/index.md). diff --git a/doc/tools/email.md b/doc/tools/email.md index e9ff88152ba..eacf63ecdcd 100644 --- a/doc/tools/email.md +++ b/doc/tools/email.md @@ -26,6 +26,9 @@ at their primary email address.  +NOTE: **Note:** +[Starting with GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/31509), email notifications can be sent only once every 10 minutes. This helps minimize performance issues. + ## Unsubscribing from emails Users can choose to unsubscribe from receiving emails from GitLab by following diff --git a/doc/topics/airgap/index.md b/doc/topics/airgap/index.md index e712e3bb6b5..854e0103a69 100644 --- a/doc/topics/airgap/index.md +++ b/doc/topics/airgap/index.md @@ -30,9 +30,75 @@ example of such a transfer: 1. Transfer images to offline environment. 1. Load transferred images into offline Docker registry. -### Example image packager script +### Using the official GitLab template -```sh +GitLab provides a [vendored template](../../ci/yaml/README.md#includetemplate) +to ease this process. + +This template should be used in a new, empty project, with a `gitlab-ci.yml` file containing: + +```yaml +include: + - template: Secure-Binaries.gitlab-ci.yml +``` + +The pipeline downloads the Docker images needed for the Security Scanners and saves them as +[job artifacts](../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../user/packages/container_registry/index.md) +of the project where the pipeline is executed. These archives can be transferred to another location +and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon. +This method requires a GitLab Runner with access to both `gitlab.com` (including +`registry.gitlab.com`) and the local offline instance. This runner must run in +[privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) +to be able to use the `docker` command inside the jobs. This runner can be installed in a DMZ or on +a bastion, and used only for this specific project. + +#### Scheduling the updates + +By default, this project's pipeline will run only once, when the `.gitlab-ci.yml` is added to the +repo. To update the GitLab security scanners and signatures, it's necessary to run this pipeline +regularly. GitLab provides a way to [schedule pipelines](../../ci/pipelines/schedules.md). For +example, you can set this up to download and store the Docker images every week. + +Some images can be updated more frequently than others. For example, the [vulnerability database](https://hub.docker.com/r/arminc/clair-db/tags) +for Container Scanning is updated daily. To update this single image, create a new Scheduled +Pipeline that runs daily and set `SECURE_BINARIES_ANALYZERS` to `clair-vulnerabilities-db`. Only +this job will be triggered, and the image will be updated daily and made available in the project +registry. + +#### Using the secure bundle created + +The project using the `Secure-Binaries.gitlab-ci.yml` template should now host all the required +images and resources needed to run GitLab Security features. + +The next step is to tell the offline instance to use these resources instead of the default ones on +`gitlab.com`. This can be done by setting the right environment variables: +`SAST_ANALYZER_IMAGE_PREFIX` for SAST analyzers, `DS_ANALYZER_IMAGE_PREFIX` for Dependency Scanning, +and so on. + +You can set these variables in the project's `.gitlab-ci.yml` files by using the bundle directly, or +in the GitLab UI at the project or group level. See the [GitLab CI/CD environment variables page](../../ci/variables/README.md#custom-environment-variables) +for more information. + +#### Variables + +The following table shows which variables you can use with the `Secure-Binaries.gitlab-ci.yml` +template: + +| VARIABLE | Description | Default value | +|-------------------------------------------|-----------------------------------------------|-----------------------------------| +| `SECURE_BINARIES_ANALYZERS` | Comma-separated list of analyzers to download | `"bandit, brakeman, gosec, and so on..."` | +| `SECURE_BINARIES_DOWNLOAD_IMAGES` | Used to disable jobs | `"true"` | +| `SECURE_BINARIES_PUSH_IMAGES` | Push files to the project registry | `"true"` | +| `SECURE_BINARIES_SAVE_ARTIFACTS` | Also save image archives as artifacts | `"false"` | +| `SECURE_BINARIES_ANALYZER_VERSION` | Default analyzer version (docker tag) | `"2"` | + +### Alternate way without the official template + +If it's not possible to follow the above method, the images can be transferred manually instead: + +#### Example image packager script + +```shell #!/bin/bash set -ux @@ -49,12 +115,12 @@ do done ``` -### Example image loader script +#### Example image loader script This example loads the images from a bastion host to an offline host. In certain configurations, physical media may be needed for such a transfer: -```sh +```shell #!/bin/bash set -ux diff --git a/doc/topics/authentication/index.md b/doc/topics/authentication/index.md index 429658984ab..e4b86a39385 100644 --- a/doc/topics/authentication/index.md +++ b/doc/topics/authentication/index.md @@ -38,7 +38,8 @@ This page gathers all the resources for the topic **Authentication** within GitL ## API - [OAuth 2 Tokens](../../api/README.md#oauth2-tokens) -- [Personal access tokens](../../api/README.md#personal-access-tokens) +- [Personal access tokens](../../api/README.md#personalproject-access-tokens) +- [Project access tokens](../../api/README.md#personalproject-access-tokens) **(CORE ONLY)** - [Impersonation tokens](../../api/README.md#impersonation-tokens) - [GitLab as an OAuth2 provider](../../api/oauth2.md#gitlab-as-an-oauth2-provider) diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 7c587ad3444..056b4c1caf4 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -2,7 +2,7 @@ While [Auto DevOps](index.md) provides great defaults to get you started, you can customize almost everything to fit your needs. Auto DevOps offers everything from custom -[buildpacks](#custom-buildpacks), to [`Dockerfiles](#custom-dockerfile), and +[buildpacks](#custom-buildpacks), to [Dockerfiles](#custom-dockerfile), and [Helm charts](#custom-helm-chart). You can even copy the complete [CI/CD configuration](#customizing-gitlab-ciyml) into your project to enable staging and canary deployments, and more. @@ -146,7 +146,7 @@ to override the default chart values by setting `HELM_UPGRADE_EXTRA_ARGS` to `-- ## Custom Helm chart per environment You can specify the use of a custom Helm chart per environment by scoping the environment variable -to the desired environment. See [Limiting environment scopes of variables](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables). +to the desired environment. See [Limiting environment scopes of variables](../../ci/variables/README.md#limit-the-environment-scopes-of-environment-variables). ## Customizing `.gitlab-ci.yml` @@ -179,7 +179,7 @@ into your project and edit it as needed. For clusters not managed by GitLab, you can customize the namespace in `.gitlab-ci.yml` by specifying -[`environment:kubernetes:namespace`](../../ci/environments.md#configuring-kubernetes-deployments). +[`environment:kubernetes:namespace`](../../ci/environments/index.md#configuring-kubernetes-deployments). For example, the following configuration overrides the namespace used for `production` deployments: @@ -227,6 +227,8 @@ If your `.gitlab-ci.yml` extends these Auto DevOps templates and override the `o `except` keywords, you must migrate your templates to use the [`rules`](../../ci/yaml/README.md#rules) syntax after the base template is migrated to use the `rules` syntax. +For users who cannot migrate just yet, you can alternatively pin your templates to +the [GitLab 12.10 based templates](https://gitlab.com/gitlab-org/auto-devops-v12-10). ## PostgreSQL database support @@ -243,23 +245,19 @@ postgres://user:password@postgres-host:postgres-port/postgres-database CAUTION: **Deprecation** The variable `AUTO_DEVOPS_POSTGRES_CHANNEL` that controls default provisioned -PostgreSQL currently defaults to `1`. This value is scheduled to change to `2` in -[GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/210499). +PostgreSQL was changed to `2` in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/210499). +To keep using the old PostgreSQL, set the `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to +`1`. The version of the chart used to provision PostgreSQL: +- Is 8.2.1 in GitLab 13.0 and later, but can be set back to 0.7.1 if needed. +- Can be set to from 0.7.1 to 8.2.1 in GitLab 12.9 and 12.10. - Is 0.7.1 in GitLab 12.8 and earlier. -- Can be set to from 0.7.1 to 8.2.1 in GitLab 12.9 and later. GitLab encourages users to [migrate their database](upgrading_postgresql.md) to the newer PostgreSQL. -To use the new PostgreSQL: - -- New projects can set the `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to `2`. -- Old projects can be upgraded by following the guide to - [upgrading PostgresSQL](upgrading_postgresql.md). - ### Using external PostgreSQL database providers While Auto DevOps provides out-of-the-box support for a PostgreSQL container for @@ -271,10 +269,9 @@ You must define environment-scoped variables for `POSTGRES_ENABLED` and `DATABASE_URL` in your project's CI/CD settings: 1. Disable the built-in PostgreSQL installation for the required environments using - scoped [environment variables](../../ci/environments.md#scoping-environments-with-specs). + scoped [environment variables](../../ci/environments/index.md#scoping-environments-with-specs). For this use case, it's likely that only `production` will need to be added to this - list. The built-in PostgreSQL setup for Review Apps and staging is sufficient, - because a high availability setup is not required. + list. The built-in PostgreSQL setup for Review Apps and staging is sufficient.  @@ -303,6 +300,7 @@ applications. |-----------------------------------------|------------------------------------| | `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. | | `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. | +| `AUTO_DEVOPS_ATOMIC_RELEASE` | As of GitLab 13.0, Auto DevOps uses [`--atomic`](https://v2.helm.sh/docs/helm/#options-43) for Helm deployments by default. Set this variable to `false` to disable the use of `--atomic` | | `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](stages.md#auto-build-using-cloud-native-buildpacks-beta). | | `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes won't prevent word splitting. [More details](#passing-arguments-to-docker-build). | | `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#passing-secrets-to-docker-build) to be passed to the `docker build` command as secrets. | @@ -318,7 +316,7 @@ applications. | `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. | | `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. | | `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. | -| `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes won't prevent word splitting. **Tip:** you can use this variable to [customize the Auto Deploy Helm chart](#custom-helm-chart) by applying custom override values with `--values my-values.yaml`. | +| `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes won't prevent word splitting. | | `INCREMENTAL_ROLLOUT_MODE` | From GitLab 11.4, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production-premium) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. | | `K8S_SECRET_*` | From GitLab 11.7, any variable prefixed with [`K8S_SECRET_`](#application-secret-variables) will be made available by Auto DevOps as environment variables to the deployed application. | | `KUBE_INGRESS_BASE_DOMAIN` | From GitLab 11.8, can be used to set a domain per cluster. See [cluster domains](../../user/project/clusters/index.md#base-domain) for more information. | @@ -329,9 +327,9 @@ applications. | `STAGING_ENABLED` | From GitLab 10.8, used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | TIP: **Tip:** -Set up the replica variables using a -[project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables) -and scale your application by only redeploying it. +After you set up your replica variables using a +[project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables), +you can scale your application by redeploying it. CAUTION: **Caution:** You should *not* scale your application using Kubernetes directly. This can @@ -350,15 +348,7 @@ The following table lists variables related to the database. | `POSTGRES_USER` | The PostgreSQL user. Defaults to `user`. Set it to use a custom username. | | `POSTGRES_PASSWORD` | The PostgreSQL password. Defaults to `testing-password`. Set it to use a custom password. | | `POSTGRES_DB` | The PostgreSQL database name. Defaults to the value of [`$CI_ENVIRONMENT_SLUG`](../../ci/variables/README.md#predefined-environment-variables). Set it to use a custom database name. | -| `POSTGRES_VERSION` | Tag for the [`postgres` Docker image](https://hub.docker.com/_/postgres) to use. Defaults to `9.6.2`. | - -### Security tools - -The following table lists variables related to security tools. - -| **Variable** | **Description** | -|-----------------------------------------|------------------------------------| -| `SAST_CONFIDENCE_LEVEL` | Minimum confidence level of security issues you want to be reported; `1` for Low, `2` for Medium, `3` for High. Defaults to `3`. | +| `POSTGRES_VERSION` | Tag for the [`postgres` Docker image](https://hub.docker.com/_/postgres) to use. Defaults to `9.6.16` for tests and deployments as of GitLab 13.0 (previously `9.6.2`). If `AUTO_DEVOPS_POSTGRES_CHANNEL` is set to `1`, deployments will use the default version `9.6.2`. | ### Disable jobs @@ -544,7 +534,7 @@ required to go from `10%` to `100%`, you can jump to whatever job you want. You can also scale down by running a lower percentage job, just before hitting `100%`. Once you get to `100%`, you can't scale down, and you'd have to roll back by redeploying the old version using the -[rollback button](../../ci/environments.md#retrying-and-rolling-back) in the +[rollback button](../../ci/environments/index.md#retrying-and-rolling-back) in the environment page. Below, you can see how the pipeline will look if the rollout or staging diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 7ed6625bea3..e7165136cf0 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -3,17 +3,25 @@ > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/37115) in GitLab 10.0. > - Generally available on GitLab 11.0. -Auto DevOps provides pre-defined CI/CD configuration which allows you to automatically detect, build, test, -deploy, and monitor your applications. Leveraging CI/CD best practices and tools, Auto DevOps aims -to simplify the setup and execution of a mature & modern software development lifecycle. +Auto DevOps provides pre-defined CI/CD configuration allowing you to automatically +detect, build, test, deploy, and monitor your applications. Leveraging CI/CD +best practices and tools, Auto DevOps aims to simplify the setup and execution +of a mature and modern software development lifecycle. ## Overview -With Auto DevOps, the software development process becomes easier to set up -as every project can have a complete workflow from verification to monitoring -with minimal configuration. Just push your code and GitLab takes -care of everything else. This makes it easier to start new projects and brings -consistency to how applications are set up throughout a company. +You can spend a lot of effort to set up the workflow and processes required to +build, deploy, and monitor your project. It gets worse when your company has +hundreds, if not thousands, of projects to maintain. With new projects +constantly starting up, the entire software development process becomes +impossibly complex to manage. + +Auto DevOps provides you a seamless software development process by +automatically detecting all dependencies and language technologies required to +test, build, package, deploy, and monitor every project with minimal +configuration. Automation enables consistency across your projects, seamless +management of processes, and faster creation of new projects: push your code, +and GitLab does the rest, improving your productivity and efficiency. For an introduction to Auto DevOps, watch [AutoDevOps in GitLab 11.0](https://youtu.be/0Tc0YYBxqi4). @@ -21,14 +29,14 @@ For an introduction to Auto DevOps, watch [AutoDevOps in GitLab 11.0](https://yo > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41729) in GitLab 11.3. -Auto DevOps is enabled by default for all projects and will attempt to run on all pipelines -in each project. This default can be enabled or disabled by an instance administrator in the +Auto DevOps is enabled by default for all projects and attempts to run on all pipelines +in each project. An instance administrator can enable or disable this default in the [Auto DevOps settings](../../user/admin_area/settings/continuous_integration.md#auto-devops-core-only). -It will be automatically disabled in individual projects on their first pipeline failure, +Auto DevOps automatically disables in individual projects on their first pipeline failure, if it has not been explicitly enabled for the project. Since [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/issues/26655), Auto DevOps -will run on pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) +runs on pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) exists. If a [CI/CD configuration file](../../ci/yaml/README.md) is present in the project, @@ -36,18 +44,21 @@ it will continue to be used, whether or not Auto DevOps is enabled. ## Quick start -If you are using GitLab.com, see the [quick start guide](quick_start_guide.md) -for how to use Auto DevOps with GitLab.com and a Kubernetes cluster on Google Kubernetes +If you're using GitLab.com, see the [quick start guide](quick_start_guide.md) +for setting up Auto DevOps with GitLab.com and a Kubernetes cluster on Google Kubernetes Engine (GKE). -If you are using a self-managed instance of GitLab, you will need to configure the +If you use a self-managed instance of GitLab, you must configure the [Google OAuth2 OmniAuth Provider](../../integration/google.md) before -you can configure a cluster on GKE. Once this is set up, you can follow the steps on the -[quick start guide](quick_start_guide.md) to get started. +configuring a cluster on GKE. After configuring the provider, you can follow +the steps in the [quick start guide](quick_start_guide.md) to get started. + +In [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) and later, it is +possible to leverage Auto DevOps to deploy to [AWS ECS](#aws-ecs). ## Comparison to application platforms and PaaS -Auto DevOps provides functionality that is often included in an application +Auto DevOps provides features often included in an application platform or a Platform as a Service (PaaS). It takes inspiration from the innovative work done by [Heroku](https://www.heroku.com/) and goes beyond it in multiple ways: @@ -60,7 +71,7 @@ in multiple ways: - Auto DevOps has more features including security testing, performance testing, and code quality testing. - Auto DevOps offers an incremental graduation path. If you need advanced customizations, - you can start modifying the templates without having to start over on a + you can start modifying the templates without starting over on a completely different platform. Review the [customizing](customize.md) documentation for more information. ## Features @@ -81,7 +92,7 @@ project in a simple and automatic way: 1. [Auto Browser Performance Testing](stages.md#auto-browser-performance-testing-premium) **(PREMIUM)** 1. [Auto Monitoring](stages.md#auto-monitoring) -As Auto DevOps relies on many different components, it's good to have a basic +As Auto DevOps relies on many different components, you should have a basic knowledge of the following: - [Kubernetes](https://kubernetes.io/docs/home/) @@ -102,101 +113,137 @@ Auto DevOps. ## Requirements -To make full use of Auto DevOps, you will need: +### Kubernetes + +To make full use of Auto DevOps with Kubernetes, you need: -- **Kubernetes** (for Auto Review Apps, Auto Deploy, and Auto Monitoring) +- **Kubernetes** (for [Auto Review Apps](stages.md#auto-review-apps), + [Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring)) - To enable deployments, you will need: + To enable deployments, you need: - 1. A [Kubernetes 1.12+ cluster](../../user/project/clusters/index.md) for the project. The easiest - way is to create a [new cluster using the GitLab UI](../../user/project/clusters/add_remove_clusters.md#create-new-cluster). - For Kubernetes 1.16+ clusters, there is some additional configuration for [Auto Deploy for Kubernetes 1.16+](stages.md#kubernetes-116). + 1. A [Kubernetes 1.12+ cluster](../../user/project/clusters/index.md) for your + project. The easiest way is to create a + [new cluster using the GitLab UI](../../user/project/clusters/add_remove_clusters.md#create-new-cluster). + For Kubernetes 1.16+ clusters, you must perform additional configuration for + [Auto Deploy for Kubernetes 1.16+](stages.md#kubernetes-116). 1. NGINX Ingress. You can deploy it to your Kubernetes cluster by installing the [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), - once you have configured GitLab's Kubernetes integration in the previous step. + after configuring GitLab's Kubernetes integration in the previous step. Alternatively, you can use the [`nginx-ingress`](https://github.com/helm/charts/tree/master/stable/nginx-ingress) Helm chart to install Ingress manually. NOTE: **Note:** - If you are using your own Ingress instead of the one provided by GitLab's managed - apps, ensure you are running at least version 0.9.0 of NGINX Ingress and + If you use your own Ingress instead of the one provided by GitLab's managed + apps, ensure you're running at least version 0.9.0 of NGINX Ingress and [enable Prometheus metrics](https://github.com/helm/charts/tree/master/stable/nginx-ingress#prometheus-metrics) - in order for the response metrics to appear. You will also have to + for the response metrics to appear. You must also [annotate](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) the NGINX Ingress deployment to be scraped by Prometheus using `prometheus.io/scrape: "true"` and `prometheus.io/port: "10254"`. -- **Base domain** (for Auto Review Apps, Auto Deploy, and Auto Monitoring) +- **Base domain** (for [Auto Review Apps](stages.md#auto-review-apps), + [Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring)) - You will need a domain configured with wildcard DNS which is going to be used - by all of your Auto DevOps applications. If you're using the + You need a domain configured with wildcard DNS, which all of your Auto DevOps + applications will use. If you're using the [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), - the URL endpoint will be automatically configured for you. + the URL endpoint is automatically configured for you. - You will then need to [specify the Auto DevOps base domain](#auto-devops-base-domain). + You must also [specify the Auto DevOps base domain](#auto-devops-base-domain). - **GitLab Runner** (for all stages) - Your Runner needs to be configured to be able to run Docker. Generally this - means using either the [Docker](https://docs.gitlab.com/runner/executors/docker.html) + Your Runner must be configured to run Docker, usually with either the + [Docker](https://docs.gitlab.com/runner/executors/docker.html) or [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html) executors, with [privileged mode enabled](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). - The Runners do not need to be installed in the Kubernetes cluster, but the - Kubernetes executor is easy to use and is automatically autoscaling. - Docker-based Runners can be configured to autoscale as well, using [Docker - Machine](https://docs.gitlab.com/runner/install/autoscaling.html). + The Runners don't need to be installed in the Kubernetes cluster, but the + Kubernetes executor is easy to use and automatically autoscales. + You can configure Docker-based Runners to autoscale as well, using + [Docker Machine](https://docs.gitlab.com/runner/install/autoscaling.html). - If you have configured GitLab's Kubernetes integration in the first step, you + If you've configured GitLab's Kubernetes integration in the first step, you can deploy it to your cluster by installing the [GitLab-managed app for GitLab Runner](../../user/clusters/applications.md#gitlab-runner). Runners should be registered as [shared Runners](../../ci/runners/README.md#registering-a-shared-runner) for the entire GitLab instance, or [specific Runners](../../ci/runners/README.md#registering-a-specific-runner) - that are assigned to specific projects (the default if you have installed the + that are assigned to specific projects (the default if you've installed the GitLab Runner managed application). -- **Prometheus** (for Auto Monitoring) +- **Prometheus** (for [Auto Monitoring](stages.md#auto-monitoring)) - To enable Auto Monitoring, you will need Prometheus installed somewhere - (inside or outside your cluster) and configured to scrape your Kubernetes cluster. - If you have configured GitLab's Kubernetes integration, you can deploy it to + To enable Auto Monitoring, you need Prometheus installed either inside or + outside your cluster, and configured to scrape your Kubernetes cluster. + If you've configured GitLab's Kubernetes integration, you can deploy it to your cluster by installing the [GitLab-managed app for Prometheus](../../user/clusters/applications.md#prometheus). The [Prometheus service](../../user/project/integrations/prometheus.md) - integration needs to be enabled for the project (or enabled as a + integration must be enabled for the project, or enabled as a [default service template](../../user/project/integrations/services_templates.md) - for the entire GitLab installation). + for the entire GitLab installation. - To get response metrics (in addition to system metrics), you need to + To get response metrics (in addition to system metrics), you must [configure Prometheus to monitor NGINX](../../user/project/integrations/prometheus_library/nginx_ingress.md#configuring-nginx-ingress-monitoring). - **cert-manager** (optional, for TLS/HTTPS) - To enable HTTPS endpoints for your application, you need to install cert-manager, - a native Kubernetes certificate management controller that helps with issuing certificates. - Installing cert-manager on your cluster will issue a certificate by - [Let’s Encrypt](https://letsencrypt.org/) and ensure that certificates are valid and up-to-date. - If you have configured GitLab's Kubernetes integration, you can deploy it to - your cluster by installing the + To enable HTTPS endpoints for your application, you must install cert-manager, + a native Kubernetes certificate management controller that helps with issuing + certificates. Installing cert-manager on your cluster issues a + [Let’s Encrypt](https://letsencrypt.org/) certificate and ensures the + certificates are valid and up-to-date. If you've configured GitLab's Kubernetes + integration, you can deploy it to your cluster by installing the [GitLab-managed app for cert-manager](../../user/clusters/applications.md#cert-manager). -If you do not have Kubernetes or Prometheus installed, then Auto Review Apps, -Auto Deploy, and Auto Monitoring will be silently skipped. +If you don't have Kubernetes or Prometheus installed, then +[Auto Review Apps](stages.md#auto-review-apps), +[Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring) +are skipped. + +After all requirements are met, you can [enable Auto DevOps](#enablingdisabling-auto-devops). + +### AWS ECS + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) in GitLab 13.0. + +You can choose to target [AWS ECS](../../ci/cloud_deployment/index.md) as a deployment platform instead of using Kubernetes. + +To get started on Auto DevOps to ECS, you'll have to add a specific Environment +Variable. To do so, follow these steps: -One all requirements are met, you can go ahead and [enable Auto DevOps](#enablingdisabling-auto-devops). +1. In your project, go to **Settings > CI / CD** and expand the **Variables** + section. + +1. Specify which AWS platform to target during the Auto DevOps deployment + by adding the `AUTO_DEVOPS_PLATFORM_TARGET` variable. + +1. Give this variable the value `ECS` before saving it. + +When you trigger a pipeline, if you have AutoDev Ops enabled and if you have correctly +[entered AWS credentials as environment variables](../../ci/cloud_deployment/index.md#deploy-your-application-to-aws-elastic-container-service-ecs), +your application will be deployed to AWS ECS. + +NOTE: **Note:** +If you have both a valid `AUTO_DEVOPS_PLATFORM_TARGET` variable and a Kubernetes cluster tied to your project, +only the deployment to Kubernetes will run. ## Auto DevOps base domain -The Auto DevOps base domain is required if you want to make use of +The Auto DevOps base domain is required to use [Auto Review Apps](stages.md#auto-review-apps), [Auto Deploy](stages.md#auto-deploy), and -[Auto Monitoring](stages.md#auto-monitoring). It can be defined in any of the following -places: - -- either under the cluster's settings, whether for [projects](../../user/project/clusters/index.md#base-domain) or [groups](../../user/group/clusters/index.md#base-domain) -- or in instance-wide settings in the **Admin Area > Settings** under the "Continuous Integration and Delivery" section +[Auto Monitoring](stages.md#auto-monitoring). You can define the base domain in +any of the following places: + +- either under the cluster's settings, whether for + [projects](../../user/project/clusters/index.md#base-domain) or + [groups](../../user/group/clusters/index.md#base-domain) +- or in instance-wide settings in **{admin}** **Admin Area > Settings** under the + **Continuous Integration and Delivery** section - or at the project level as a variable: `KUBE_INGRESS_BASE_DOMAIN` - or at the group level as a variable: `KUBE_INGRESS_BASE_DOMAIN`. @@ -204,55 +251,57 @@ The base domain variable `KUBE_INGRESS_BASE_DOMAIN` follows the same order of pr as other environment [variables](../../ci/variables/README.md#priority-of-environment-variables). TIP: **Tip:** -If you're using the [GitLab managed app for Ingress](../../user/clusters/applications.md#ingress), -the URL endpoint should be automatically configured for you. All you have to do +If you use the [GitLab managed app for Ingress](../../user/clusters/applications.md#ingress), +the URL endpoint should be automatically configured for you. All you must do is use its value for the `KUBE_INGRESS_BASE_DOMAIN` variable. NOTE: **Note:** `AUTO_DEVOPS_DOMAIN` was [deprecated in GitLab 11.8](https://gitlab.com/gitlab-org/gitlab-foss/issues/52363) -and replaced with `KUBE_INGRESS_BASE_DOMAIN`. It was removed in +and replaced with `KUBE_INGRESS_BASE_DOMAIN`, and removed in [GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-foss/issues/56959). -A wildcard DNS A record matching the base domain(s) is required, for example, -given a base domain of `example.com`, you'd need a DNS entry like: +Auto DevOps requires a wildcard DNS A record matching the base domain(s). For +a base domain of `example.com`, you'd need a DNS entry like: -```text +```plaintext *.example.com 3600 A 1.2.3.4 ``` -In this case, `example.com` is the domain name under which the deployed apps will be served, -and `1.2.3.4` is the IP address of your load balancer; generally NGINX -([see requirements](#requirements)). How to set up the DNS record is beyond -the scope of this document; you should check with your DNS provider. +In this case, the deployed applications are served from `example.com`, and `1.2.3.4` +is the IP address of your load balancer; generally NGINX ([see requirements](#requirements)). +Setting up the DNS record is beyond the scope of this document; check with your +DNS provider for information. -Alternatively you can use free public services like [nip.io](https://nip.io) -which provide automatic wildcard DNS without any configuration. Just set the -Auto DevOps base domain to `1.2.3.4.nip.io`. +Alternatively, you can use free public services like [nip.io](https://nip.io) +which provide automatic wildcard DNS without any configuration. For [nip.io](https://nip.io), +set the Auto DevOps base domain to `1.2.3.4.nip.io`. -Once set up, all requests will hit the load balancer, which in turn will route -them to the Kubernetes pods that run your application(s). +After completing setup, all requests hit the load balancer, which routes requests +to the Kubernetes pods running your application. ## Enabling/Disabling Auto DevOps -When first using Auto DevOps, review the [requirements](#requirements) to ensure all necessary components to make -full use of Auto DevOps are available. If this is your fist time, we recommend you follow the -[quick start guide](quick_start_guide.md). +When first using Auto DevOps, review the [requirements](#requirements) to ensure +all the necessary components to make full use of Auto DevOps are available. First-time +users should follow the [quick start guide](quick_start_guide.md). -GitLab.com users can enable/disable Auto DevOps at the project-level only. Self-managed users -can enable/disable Auto DevOps at the project-level, group-level or instance-level. +GitLab.com users can enable or disable Auto DevOps only at the project level. +Self-managed users can enable or disable Auto DevOps at the project, group, or +instance level. ### At the project level -If enabling, check that your project doesn't have a `.gitlab-ci.yml`, or if one exists, remove it. +If enabling, check that your project does not have a `.gitlab-ci.yml`, or if one exists, remove it. -1. Go to your project's **Settings > CI/CD > Auto DevOps**. -1. Toggle the **Default to Auto DevOps pipeline** checkbox (checked to enable, unchecked to disable) -1. When enabling, it's optional but recommended to add in the [base domain](#auto-devops-base-domain) - that will be used by Auto DevOps to [deploy your application](stages.md#auto-deploy) +1. Go to your project's **{settings}** **Settings > CI/CD > Auto DevOps**. +1. Select the **Default to Auto DevOps pipeline** checkbox to enable it. +1. (Optional, but recommended) When enabling, you can add in the + [base domain](#auto-devops-base-domain) Auto DevOps uses to + [deploy your application](stages.md#auto-deploy), and choose the [deployment strategy](#deployment-strategy). 1. Click **Save changes** for the changes to take effect. -When the feature has been enabled, an Auto DevOps pipeline is triggered on the default branch. +After enabling the feature, an Auto DevOps pipeline is triggered on the default branch. ### At the group level @@ -260,48 +309,50 @@ When the feature has been enabled, an Auto DevOps pipeline is triggered on the d Only administrators and group owners can enable or disable Auto DevOps at the group level. -To enable or disable Auto DevOps at the group-level: +When enabling or disabling Auto DevOps at group level, group configuration is +implicitly used for the subgroups and projects inside that group, unless Auto DevOps +is specifically enabled or disabled on the subgroup or project. -1. Go to group's **Settings > CI/CD > Auto DevOps** page. -1. Toggle the **Default to Auto DevOps pipeline** checkbox (checked to enable, unchecked to disable). -1. Click **Save changes** button for the changes to take effect. +To enable or disable Auto DevOps at the group level: -When enabling or disabling Auto DevOps at group-level, group configuration will be implicitly used for -the subgroups and projects inside that group, unless Auto DevOps is specifically enabled or disabled on -the subgroup or project. +1. Go to your group's **{settings}** **Settings > CI/CD > Auto DevOps** page. +1. Select the **Default to Auto DevOps pipeline** checkbox to enable it. +1. Click **Save changes** for the changes to take effect. ### At the instance level (Administrators only) Even when disabled at the instance level, group owners and project maintainers can still enable Auto DevOps at the group and project level, respectively. -1. Go to **Admin Area > Settings > Continuous Integration and Deployment**. -1. Toggle the checkbox labeled **Default to Auto DevOps pipeline for all projects**. -1. If enabling, optionally set up the Auto DevOps [base domain](#auto-devops-base-domain) which will be used for Auto Deploy and Auto Review Apps. +1. Go to **{admin}** **Admin Area > Settings > Continuous Integration and Deployment**. +1. Select **Default to Auto DevOps pipeline for all projects** to enable it. +1. (Optional) You can set up the Auto DevOps [base domain](#auto-devops-base-domain), + for Auto Deploy and Auto Review Apps to use. 1. Click **Save changes** for the changes to take effect. ### Enable for a percentage of projects -There is also a feature flag to enable Auto DevOps by default to your chosen percentage of projects. +You can use a feature flag to enable Auto DevOps by default to your desired percentage +of projects. From the console, enter the following command, replacing `10` with +your desired percentage: -This can be enabled from the console with the following, which uses the example of 10%: - -`Feature.get(:force_autodevops_on_by_default).enable_percentage_of_actors(10)` +```ruby +Feature.get(:force_autodevops_on_by_default).enable_percentage_of_actors(10) +``` ### Deployment strategy > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/38542) in GitLab 11.0. You can change the deployment strategy used by Auto DevOps by going to your -project's **Settings > CI/CD > Auto DevOps**. - -The available options are: +project's **{settings}** **Settings > CI/CD > Auto DevOps**. The following options +are available: - **Continuous deployment to production**: Enables [Auto Deploy](stages.md#auto-deploy) with `master` branch directly deployed to production. - **Continuous deployment to production using timed incremental rollout**: Sets the [`INCREMENTAL_ROLLOUT_MODE`](customize.md#timed-incremental-rollout-to-production-premium) variable - to `timed`, and production deployment will be executed with a 5 minute delay between + to `timed`. Production deployments execute with a 5 minute delay between each increment in rollout. - **Automatic deployment to staging, manual deployment to production**: Sets the [`STAGING_ENABLED`](customize.md#deploy-policy-for-staging-and-production-environments) and @@ -313,63 +364,61 @@ The available options are: ## Using multiple Kubernetes clusters **(PREMIUM)** -When using Auto DevOps, you may want to deploy different environments to -different Kubernetes clusters. This is possible due to the 1:1 connection that -[exists between them](../../user/project/clusters/index.md#multiple-kubernetes-clusters-premium). +When using Auto DevOps, you can deploy different environments to +different Kubernetes clusters, due to the 1:1 connection +[existing between them](../../user/project/clusters/index.md#multiple-kubernetes-clusters-premium). -In the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) (used behind the scenes by Auto DevOps), there -are currently 3 defined environment names that you need to know: +The [template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) +used by Auto DevOps currently defines 3 environment names: - `review/` (every environment starting with `review/`) - `staging` - `production` -Those environments are tied to jobs that use [Auto Deploy](stages.md#auto-deploy), so -except for the environment scope, they would also need to have a different -domain they would be deployed to. This is why you need to define a separate -`KUBE_INGRESS_BASE_DOMAIN` variable for all the above -[based on the environment](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables). +Those environments are tied to jobs using [Auto Deploy](stages.md#auto-deploy), so +except for the environment scope, they must have a different deployment domain. +You must define a separate `KUBE_INGRESS_BASE_DOMAIN` variable for each of the above +[based on the environment](../../ci/variables/README.md#limit-the-environment-scopes-of-environment-variables). -The following table is an example of how the three different clusters would -be configured. +The following table is an example of how to configure the three different clusters: | Cluster name | Cluster environment scope | `KUBE_INGRESS_BASE_DOMAIN` variable value | Variable environment scope | Notes | |--------------|---------------------------|-------------------------------------------|----------------------------|---| -| review | `review/*` | `review.example.com` | `review/*` | The review cluster which will run all [Review Apps](../../ci/review_apps/index.md). `*` is a wildcard, which means it will be used by every environment name starting with `review/`. | -| staging | `staging` | `staging.example.com` | `staging` | (Optional) The staging cluster which will run the deployments of the staging environments. You need to [enable it first](customize.md#deploy-policy-for-staging-and-production-environments). | -| production | `production` | `example.com` | `production` | The production cluster which will run the deployments of the production environment. You can use [incremental rollouts](customize.md#incremental-rollout-to-production-premium). | +| review | `review/*` | `review.example.com` | `review/*` | The review cluster which runs all [Review Apps](../../ci/review_apps/index.md). `*` is a wildcard, used by every environment name starting with `review/`. | +| staging | `staging` | `staging.example.com` | `staging` | (Optional) The staging cluster which runs the deployments of the staging environments. You must [enable it first](customize.md#deploy-policy-for-staging-and-production-environments). | +| production | `production` | `example.com` | `production` | The production cluster which runs the production environment deployments. You can use [incremental rollouts](customize.md#incremental-rollout-to-production-premium). | To add a different cluster for each environment: -1. Navigate to your project's **Operations > Kubernetes** and create the Kubernetes clusters - with their respective environment scope as described from the table above. +1. Navigate to your project's **{cloud-gear}** **Operations > Kubernetes**. +1. Create the Kubernetes clusters with their respective environment scope, as + described from the table above.  -1. After the clusters are created, navigate to each one and install Helm Tiller +1. After creating the clusters, navigate to each cluster and install Helm Tiller and Ingress. Wait for the Ingress IP address to be assigned. -1. Make sure you have [configured your DNS](#auto-devops-base-domain) with the +1. Make sure you've [configured your DNS](#auto-devops-base-domain) with the specified Auto DevOps domains. -1. Navigate to each cluster's page, through **Operations > Kubernetes**, +1. Navigate to each cluster's page, through **{cloud-gear}** **Operations > Kubernetes**, and add the domain based on its Ingress IP address. -Now that all is configured, you can test your setup by creating a merge request -and verifying that your app is deployed as a review app in the Kubernetes +After completing configuration, you can test your setup by creating a merge request +and verifying your application is deployed as a Review App in the Kubernetes cluster with the `review/*` environment scope. Similarly, you can check the other environments. ## Currently supported languages Note that not all buildpacks support Auto Test yet, as it's a relatively new -enhancement. All of Heroku's [officially supported -languages](https://devcenter.heroku.com/articles/heroku-ci#currently-supported-languages) -support it, and some third-party buildpacks as well e.g., Go, Node, Java, PHP, -Python, Ruby, Gradle, Scala, and Elixir all support Auto Test, but notably the -multi-buildpack does not. +enhancement. All of Heroku's +[officially supported languages](https://devcenter.heroku.com/articles/heroku-ci#supported-languages) +support Auto Test. The languages supported by Heroku's Herokuish buildpacks all +support Auto Test, but notably the multi-buildpack does not. As of GitLab 10.0, the supported buildpacks are: -```text +```plaintext - heroku-buildpack-multi v1.0.0 - heroku-buildpack-ruby v168 - heroku-buildpack-nodejs v99 @@ -385,24 +434,27 @@ As of GitLab 10.0, the supported buildpacks are: - buildpack-nginx v8 ``` +If your application needs a buildpack that is not in the above list, you +might want to use a [custom buildpack](customize.md#custom-buildpacks). + ## Limitations The following restrictions apply. ### Private registry support -There is no documented way of using private container registry with Auto DevOps. -We strongly advise using GitLab Container Registry with Auto DevOps in order to +No documented way of using private container registry with Auto DevOps exists. +We strongly advise using GitLab Container Registry with Auto DevOps to simplify configuration and prevent any unforeseen issues. ### Installing Helm behind a proxy -GitLab does not yet support installing [Helm as a GitLab-managed App](../../user/clusters/applications.md#helm) when -behind a proxy. Users who wish to do so must inject their proxy settings -into the installation pods at runtime, for example by using a +GitLab does not support installing [Helm as a GitLab-managed App](../../user/clusters/applications.md#helm) when +behind a proxy. Users who want to do so must inject their proxy settings +into the installation pods at runtime, such as by using a [`PodPreset`](https://kubernetes.io/docs/concepts/workloads/pods/podpreset/): -```yml +```yaml apiVersion: settings.k8s.io/v1alpha1 kind: PodPreset metadata: @@ -418,28 +470,52 @@ spec: ## Troubleshooting -- Auto Build and Auto Test may fail to detect your language or framework with the - following error: - - ```plaintext - Step 5/11 : RUN /bin/herokuish buildpack build - ---> Running in eb468cd46085 - -----> Unable to select a buildpack - The command '/bin/sh -c /bin/herokuish buildpack build' returned a non-zero code: 1 - ``` - - The following are possible reasons: - - - Your application may be missing the key files the buildpack is looking for. For - example, for Ruby applications you must have a `Gemfile` to be properly detected, - even though it is possible to write a Ruby app without a `Gemfile`. - - There may be no buildpack for your application. Try specifying a - [custom buildpack](customize.md#custom-buildpacks). -- Auto Test may fail because of a mismatch between testing frameworks. In this - case, you may need to customize your `.gitlab-ci.yml` with your test commands. -- Auto Deploy will fail if GitLab can not create a Kubernetes namespace and - service account for your project. For help debugging this issue, see - [Troubleshooting failed deployment jobs](../../user/project/clusters/index.md#troubleshooting). +### Unable to select a buildpack + +Auto Build and Auto Test may fail to detect your language or framework with the +following error: + +```plaintext +Step 5/11 : RUN /bin/herokuish buildpack build + ---> Running in eb468cd46085 + -----> Unable to select a buildpack +The command '/bin/sh -c /bin/herokuish buildpack build' returned a non-zero code: 1 +``` + +The following are possible reasons: + +- Your application may be missing the key files the buildpack is looking for. + Ruby applications require a `Gemfile` to be properly detected, + even though it's possible to write a Ruby app without a `Gemfile`. +- No buildpack may exist for your application. Try specifying a + [custom buildpack](customize.md#custom-buildpacks). + +### Mismatch between testing frameworks + +Auto Test may fail because of a mismatch between testing frameworks. In this +case, you may need to customize your `.gitlab-ci.yml` with your test commands. + +### Pipeline that extends Auto DevOps with only / except fails + +If your pipeline fails with the following message: + +```plaintext +Found errors in your .gitlab-ci.yml: + + jobs:test config key may not be used with `rules`: only +``` + +This error appears when the included job’s rules configuration has been overridden with the `only` or `except` syntax. +To fix this issue, you must either: + +- Transition your `only/except` syntax to rules. +- (Temporarily) Pin your templates to the [GitLab 12.10 based templates](https://gitlab.com/gitlab-org/auto-devops-v12-10). + +### Failure to create a Kubernetes namespace + +Auto Deploy will fail if GitLab can't create a Kubernetes namespace and +service account for your project. For help debugging this issue, see +[Troubleshooting failed deployment jobs](../../user/project/clusters/index.md#troubleshooting). ## Development guides diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 53d5e664bc1..859219689f9 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -88,7 +88,7 @@ to deploy this project to. [Cloud Run](../../user/project/clusters/add_gke_clusters.md#cloud-run-for-anthos), Istio, and HTTP Load Balancing add-ons for this cluster. - **GitLab-managed cluster** - Select this checkbox to - [allow GitLab to manage namespace and service accounts](../..//user/project/clusters/index.md#gitlab-managed-clusters) for this cluster. + [allow GitLab to manage namespace and service accounts](../../user/project/clusters/index.md#gitlab-managed-clusters) for this cluster. 1. Click **Create Kubernetes cluster**. @@ -215,12 +215,12 @@ you to common environment tasks: about the Kubernetes cluster and how the application affects it in terms of memory usage, CPU usage, and latency - **Deploy to** (**{play}** **{angle-down}**) - Displays a list of environments you can deploy to -- **Terminal** (**{terminal}**) - Opens a [web terminal](../../ci/environments.md#web-terminals) +- **Terminal** (**{terminal}**) - Opens a [web terminal](../../ci/environments/index.md#web-terminals) session inside the container where the application is running - **Re-deploy to environment** (**{repeat}**) - For more information, see - [Retrying and rolling back](../../ci/environments.md#retrying-and-rolling-back) + [Retrying and rolling back](../../ci/environments/index.md#retrying-and-rolling-back) - **Stop environment** (**{stop}**) - For more information, see - [Stopping an environment](../../ci/environments.md#stopping-an-environment) + [Stopping an environment](../../ci/environments/index.md#stopping-an-environment) GitLab displays the [Deploy Board](../../user/project/deploy_boards.md) below the environment's information, with squares representing pods in your diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 66b76dcc05a..8c56a87ba30 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -1,47 +1,50 @@ # Stages of Auto DevOps -The following sections describe the stages of Auto DevOps. Read them carefully -to understand how each one works. +The following sections describe the stages of [Auto DevOps](index.md). +Read them carefully to understand how each one works. ## Auto Build Auto Build creates a build of the application using an existing `Dockerfile` or -Heroku buildpacks. - -Either way, the resulting Docker image is automatically pushed to the -[Container Registry](../../user/packages/container_registry/index.md) and tagged with the commit SHA or tag. +Heroku buildpacks. The resulting Docker image is pushed to the +[Container Registry](../../user/packages/container_registry/index.md), and tagged +with the commit SHA or tag. ### Auto Build using a Dockerfile -If a project's repository contains a `Dockerfile` at its root, Auto Build will use +If a project's repository contains a `Dockerfile` at its root, Auto Build uses `docker build` to create a Docker image. -If you are also using Auto Review Apps and Auto Deploy and choose to provide -your own `Dockerfile`, make sure you expose your application to port -`5000` as this is the port assumed by the -[default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). Alternatively you can override the default values by [customizing the Auto Deploy Helm chart](customize.md#custom-helm-chart) +If you're also using Auto Review Apps and Auto Deploy, and you choose to provide +your own `Dockerfile`, you must either: + +- Expose your application to port `5000`, as the + [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app) + assumes this port is available. +- Override the default values by + [customizing the Auto Deploy Helm chart](customize.md#custom-helm-chart). ### Auto Build using Heroku buildpacks -Auto Build builds an application using a project's `Dockerfile` if present, or -otherwise it will use [Herokuish](https://github.com/gliderlabs/herokuish) +Auto Build builds an application using a project's `Dockerfile` if present. If no +`Dockerfile` is present, it uses [Herokuish](https://github.com/gliderlabs/herokuish) and [Heroku buildpacks](https://devcenter.heroku.com/articles/buildpacks) -to automatically detect and build the application into a Docker image. +to detect and build the application into a Docker image. -Each buildpack requires certain files to be in your project's repository for -Auto Build to successfully build your application. For example, the following -files are required at the root of your application's repository, depending on -the language: +Each buildpack requires your project's repository to contain certain files for +Auto Build to build your application successfully. For example, your application's +root directory must contain the appropriate file for your application's +language: -- A `Pipfile` or `requirements.txt` file for Python projects. -- A `Gemfile` or `Gemfile.lock` file for Ruby projects. +- For Python projects, a `Pipfile` or `requirements.txt` file. +- For Ruby projects, a `Gemfile` or `Gemfile.lock` file. For the requirements of other languages and frameworks, read the -[buildpacks docs](https://devcenter.heroku.com/articles/buildpacks#officially-supported-buildpacks). +[Heroku buildpacks documentation](https://devcenter.heroku.com/articles/buildpacks#officially-supported-buildpacks). TIP: **Tip:** If Auto Build fails despite the project meeting the buildpack requirements, set -a project variable `TRACE=true` to enable verbose logging, which may help to +a project variable `TRACE=true` to enable verbose logging, which may help you troubleshoot. ### Auto Build using Cloud Native Buildpacks (beta) @@ -73,13 +76,13 @@ yet part of the Cloud Native Buildpack specification. For more information, see ## Auto Test -Auto Test automatically runs the appropriate tests for your application using -[Herokuish](https://github.com/gliderlabs/herokuish) and [Heroku -buildpacks](https://devcenter.heroku.com/articles/buildpacks) by analyzing +Auto Test runs the appropriate tests for your application using +[Herokuish](https://github.com/gliderlabs/herokuish) and +[Heroku buildpacks](https://devcenter.heroku.com/articles/buildpacks) by analyzing your project to detect the language and framework. Several languages and frameworks are detected automatically, but if your language is not detected, -you may succeed with a [custom buildpack](customize.md#custom-buildpacks). Check the -[currently supported languages](index.md#currently-supported-languages). +you may be able to create a [custom buildpack](customize.md#custom-buildpacks). +Check the [currently supported languages](index.md#currently-supported-languages). Auto Test uses tests you already have in your application. If there are no tests, it's up to you to add them. @@ -88,12 +91,10 @@ tests, it's up to you to add them. Auto Code Quality uses the [Code Quality image](https://gitlab.com/gitlab-org/ci-cd/codequality) to run -static analysis and other code checks on the current code. The report is -created, and is uploaded as an artifact which you can later download and check -out. - -Any differences between the source and target branches are also -[shown in the merge request widget](../../user/project/merge_requests/code_quality.md). +static analysis and other code checks on the current code. After creating the +report, it's uploaded as an artifact which you can later download and check +out. The merge request widget also displays any +[differences between the source and target branches](../../user/project/merge_requests/code_quality.md). ## Auto SAST **(ULTIMATE)** @@ -101,14 +102,17 @@ Any differences between the source and target branches are also Static Application Security Testing (SAST) uses the [SAST Docker image](https://gitlab.com/gitlab-org/security-products/sast) to run static -analysis on the current code and checks for potential security issues. The -Auto SAST stage will be skipped on licenses other than Ultimate and requires GitLab Runner 11.5 or above. +analysis on the current code, and checks for potential security issues. The +Auto SAST stage will be skipped on licenses other than +[Ultimate](https://about.gitlab.com/pricing/), and requires +[GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or above. -Once the report is created, it's uploaded as an artifact which you can later download and -check out. +After creating the report, it's uploaded as an artifact which you can later +download and check out. The merge request widget also displays any security +warnings. -Any security warnings are also shown in the merge request widget. Read more how -[SAST works](../../user/application_security/sast/index.md). +To learn more about [how SAST works](../../user/application_security/sast/index.md), +see the documentation. ## Auto Dependency Scanning **(ULTIMATE)** @@ -116,16 +120,17 @@ Any security warnings are also shown in the merge request widget. Read more how Dependency Scanning uses the [Dependency Scanning Docker image](https://gitlab.com/gitlab-org/security-products/dependency-scanning) -to run analysis on the project dependencies and checks for potential security issues. -The Auto Dependency Scanning stage will be skipped on licenses other than Ultimate -and requires GitLab Runner 11.5 or above. +to run analysis on the project dependencies and check for potential security issues. +The Auto Dependency Scanning stage is skipped on licenses other than +[Ultimate](https://about.gitlab.com/pricing/) and requires +[GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or above. -Once the -report is created, it's uploaded as an artifact which you can later download and -check out. +After creating the report, it's uploaded as an artifact which you can later download and +check out. The merge request widget displays any security warnings detected, -Any security warnings are also shown in the merge request widget. Read more about -[Dependency Scanning](../../user/application_security/dependency_scanning/index.md). +To learn more about +[Dependency Scanning](../../user/application_security/dependency_scanning/index.md), +see the documentation. ## Auto License Compliance **(ULTIMATE)** @@ -134,60 +139,57 @@ Any security warnings are also shown in the merge request widget. Read more abou License Compliance uses the [License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) to search the project dependencies for their license. The Auto License Compliance stage -will be skipped on licenses other than Ultimate. +is skipped on licenses other than [Ultimate](https://about.gitlab.com/pricing/). -Once the -report is created, it's uploaded as an artifact which you can later download and -check out. +After creating the report, it's uploaded as an artifact which you can later download and +check out. The merge request displays any detected licenses. -Any licenses are also shown in the merge request widget. Read more how -[License Compliance works](../../user/compliance/license_compliance/index.md). +To learn more about +[License Compliance](../../user/compliance/license_compliance/index.md), see the +documentation. ## Auto Container Scanning **(ULTIMATE)** > Introduced in GitLab 10.4. -Vulnerability Static Analysis for containers uses -[Clair](https://github.com/quay/clair) to run static analysis on a -Docker image and checks for potential security issues. The Auto Container Scanning stage -will be skipped on licenses other than Ultimate. +Vulnerability Static Analysis for containers uses [Clair](https://github.com/quay/clair) +to check for potential security issues on Docker images. The Auto Container Scanning +stage is skipped on licenses other than [Ultimate](https://about.gitlab.com/pricing/). -Once the report is -created, it's uploaded as an artifact which you can later download and -check out. +After creating the report, it's uploaded as an artifact which you can later download and +check out. The merge request displays any detected security issues. -Any security warnings are also shown in the merge request widget. Read more how -[Container Scanning works](../../user/application_security/container_scanning/index.md). +To learn more about +[Container Scanning](../../user/application_security/container_scanning/index.md), +see the documentation. ## Auto Review Apps -This is an optional step, since many projects do not have a Kubernetes cluster -available. If the [requirements](index.md#requirements) are not met, the job will -silently be skipped. +This is an optional step, since many projects don't have a Kubernetes cluster +available. If the [requirements](index.md#requirements) are not met, the job is +silently skipped. [Review Apps](../../ci/review_apps/index.md) are temporary application environments based on the branch's code so developers, designers, QA, product managers, and other reviewers can actually see and interact with code changes as part of the review process. Auto Review Apps create a Review App for each branch. -Auto Review Apps will deploy your app to your Kubernetes cluster only. When no cluster -is available, no deployment will occur. +Auto Review Apps deploy your application to your Kubernetes cluster only. If no cluster +is available, no deployment occurs. -The Review App will have a unique URL based on the project ID, the branch or tag -name, and a unique number, combined with the Auto DevOps base domain. For -example, `13083-review-project-branch-123456.example.com`. A link to the Review App shows -up in the merge request widget for easy discovery. When the branch or tag is deleted, -for example after the merge request is merged, the Review App will automatically -be deleted. +The Review App has a unique URL based on a combination of the project ID, the branch +or tag name, a unique number, and the Auto DevOps base domain, such as +`13083-review-project-branch-123456.example.com`. The merge request widget displays +a link to the Review App for easy discovery. When the branch or tag is deleted, +such as after merging a merge request, the Review App is also deleted. Review apps are deployed using the [auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with -Helm, which can be [customized](customize.md#custom-helm-chart). The app will be deployed into the [Kubernetes -namespace](../../user/project/clusters/index.md#deployment-variables) +Helm, which you can [customize](customize.md#custom-helm-chart). The application deploys +into the [Kubernetes namespace](../../user/project/clusters/index.md#deployment-variables) for the environment. -Since GitLab 11.4, a [local -Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is +Since GitLab 11.4, [local Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is used. Previous versions of GitLab had a Tiller installed in the project namespace. @@ -196,52 +198,64 @@ Your apps should *not* be manipulated outside of Helm (using Kubernetes directly This can cause confusion with Helm not detecting the change and subsequent deploys with Auto DevOps can undo your changes. Also, if you change something and want to undo it by deploying again, Helm may not detect that anything changed -in the first place, and thus not realize that it needs to re-apply the old config. +in the first place, and thus not realize that it needs to re-apply the old configuration. ## Auto DAST **(ULTIMATE)** > Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. -Dynamic Application Security Testing (DAST) uses the -popular open source tool [OWASP ZAProxy](https://github.com/zaproxy/zaproxy) -to perform an analysis on the current code and checks for potential security -issues. The Auto DAST stage will be skipped on licenses other than Ultimate. +Dynamic Application Security Testing (DAST) uses the popular open source tool +[OWASP ZAProxy](https://github.com/zaproxy/zaproxy) to analyze the current code +and check for potential security issues. The Auto DAST stage is skipped on +licenses other than [Ultimate](https://about.gitlab.com/pricing/). -Once the DAST scan is complete, any security warnings are shown -on the [Security Dashboard](../../user/application_security/security_dashboard/index.md) -and the Merge Request Widget. Read how -[DAST works](../../user/application_security/dast/index.md). +- On your default branch, DAST scans an application deployed specifically for that purpose + unless you [override the target branch](#overriding-the-dast-target). + The app is deleted after DAST has run. +- On feature branches, DAST scans the [review app](#auto-review-apps). -On your default branch, DAST scans an app deployed specifically for that purpose. -The app is deleted after DAST has run. +After the DAST scan completes, any security warnings are displayed +on the [Security Dashboard](../../user/application_security/security_dashboard/index.md) +and the merge request widget. -On feature branches, DAST scans the [review app](#auto-review-apps). +To learn more about +[Dynamic Application Security Testing](../../user/application_security/dast/index.md), +see the documentation. ### Overriding the DAST target To use a custom target instead of the auto-deployed review apps, set a `DAST_WEBSITE` environment variable to the URL for DAST to scan. -NOTE: **Note:** -If [DAST Full Scan](../../user/application_security/dast/index.md#full-scan) is enabled, it is strongly advised **not** +DANGER: **Danger:** +If [DAST Full Scan](../../user/application_security/dast/index.md#full-scan) is +enabled, GitLab strongly advises **not** to set `DAST_WEBSITE` to any staging or production environment. DAST Full Scan -actively attacks the target, which can take down the application and lead to +actively attacks the target, which can take down your application and lead to data loss or corruption. ### Disabling Auto DAST -DAST can be disabled: +You can disable DAST: - On all branches by setting the `DAST_DISABLED` environment variable to `"true"`. -- Only on the default branch by setting the `DAST_DISABLED_FOR_DEFAULT_BRANCH` environment variable to `"true"`. +- Only on the default branch by setting the `DAST_DISABLED_FOR_DEFAULT_BRANCH` + environment variable to `"true"`. +- Only on feature branches by setting `REVIEW_DISABLED` environment variable to + `"true"`. This also disables the Review App. ## Auto Browser Performance Testing **(PREMIUM)** > Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. -Auto Browser Performance Testing utilizes the [Sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/) to measure the performance of a web page. A JSON report is created and uploaded as an artifact, which includes the overall performance score for each page. By default, the root page of Review and Production environments will be tested. If you would like to add additional URL's to test, simply add the paths to a file named `.gitlab-urls.txt` in the root directory, one per line. For example: +Auto Browser Performance Testing measures the performance of a web page with the +[Sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/), +creates a JSON report including the overall performance score for each page, and +uploads the report as an artifact. By default, it tests the root page of your Review and +Production environments. If you want to test additional URLs, add the paths to a +file named `.gitlab-urls.txt` in the root directory, one file per line. For example: -```text +```plaintext / /features /direction @@ -252,30 +266,31 @@ Any performance differences between the source and target branches are also ## Auto Deploy -This is an optional step, since many projects do not have a Kubernetes cluster -available. If the [requirements](index.md#requirements) are not met, the job will -silently be skipped. +This is an optional step, since many projects don't have a Kubernetes cluster +available. If the [requirements](index.md#requirements) are not met, the job is skipped. After a branch or merge request is merged into the project's default branch (usually `master`), Auto Deploy deploys the application to a `production` environment in the Kubernetes cluster, with a namespace based on the project name and unique -project ID, for example `project-4321`. +project ID, such as `project-4321`. -Auto Deploy doesn't include deployments to staging or canary by default, but the -[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to -enable them. +Auto Deploy does not include deployments to staging or canary environments by +default, but the +[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) +contains job definitions for these tasks if you want to enable them. -You can make use of [environment variables](customize.md#environment-variables) to automatically -scale your pod replicas and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](customize.md#custom-helm-chart). +You can use [environment variables](customize.md#environment-variables) to automatically +scale your pod replicas, and to apply custom arguments to the Auto DevOps `helm upgrade` +commands. This is an easy way to +[customize the Auto Deploy Helm chart](customize.md#custom-helm-chart). -Apps are deployed using the -[auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with -Helm. The app will be deployed into the [Kubernetes -namespace](../../user/project/clusters/index.md#deployment-variables) +Helm uses the [auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) +chart to deploy the application into the +[Kubernetes namespace](../../user/project/clusters/index.md#deployment-variables) for the environment. -Since GitLab 11.4, a [local -Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is +Since GitLab 11.4, a +[local Tiller](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22036) is used. Previous versions of GitLab had a Tiller installed in the project namespace. @@ -284,76 +299,85 @@ Your apps should *not* be manipulated outside of Helm (using Kubernetes directly This can cause confusion with Helm not detecting the change and subsequent deploys with Auto DevOps can undo your changes. Also, if you change something and want to undo it by deploying again, Helm may not detect that anything changed -in the first place, and thus not realize that it needs to re-apply the old config. +in the first place, and thus not realize that it needs to re-apply the old configuration. + +### GitLab deploy tokens > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19507) in GitLab 11.0. -For internal and private projects a [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token) -will be automatically created, when Auto DevOps is enabled and the Auto DevOps settings are saved. This Deploy Token -can be used for permanent access to the registry. When the GitLab Deploy Token has been manually revoked, it won't be automatically created. +[GitLab Deploy Tokens](../../user/project/deploy_tokens/index.md#gitlab-deploy-token) +are created for internal and private projects when Auto DevOps is enabled, and the +Auto DevOps settings are saved. You can use a Deploy Token for permanent access to +the registry. After you manually revoke the GitLab Deploy Token, it won't be +automatically created. + +If the GitLab Deploy Token can't be found, `CI_REGISTRY_PASSWORD` is +used. -If the GitLab Deploy Token cannot be found, `CI_REGISTRY_PASSWORD` is -used. Note that `CI_REGISTRY_PASSWORD` is only valid during deployment. -This means that Kubernetes will be able to successfully pull the -container image during deployment but in cases where the image needs to -be pulled again, e.g. after pod eviction, Kubernetes will fail to do so -as it will be attempting to fetch the image using -`CI_REGISTRY_PASSWORD`. +NOTE: **Note:** +`CI_REGISTRY_PASSWORD` is only valid during deployment. Kubernetes will be able +to successfully pull the container image during deployment, but if the image must +be pulled again, such as after pod eviction, Kubernetes will fail to do so +as it attempts to fetch the image using `CI_REGISTRY_PASSWORD`. ### Kubernetes 1.16+ > - [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/51) in GitLab 12.8. > - Support for deploying a PostgreSQL version that supports Kubernetes 1.16+ was [introduced](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/49) in GitLab 12.9. +> - Supported out of the box for new deployments as of GitLab 13.0. CAUTION: **Deprecation** -The default value of `extensions/v1beta1` for the `deploymentApiVersion` setting is -deprecated, and is scheduled to be changed to a new default of `apps/v1` in -[GitLab 13.0](https://gitlab.com/gitlab-org/charts/auto-deploy-app/issues/47). +The default value for the `deploymentApiVersion` setting was changed from +`extensions/v1beta` to `apps/v1` in [GitLab 13.0](https://gitlab.com/gitlab-org/charts/auto-deploy-app/issues/47). -In Kubernetes 1.16 onwards, a number of [APIs were removed](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/), +In Kubernetes 1.16 and later, a number of +[APIs were removed](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/), including support for `Deployment` in the `extensions/v1beta1` version. -To use Auto Deploy on a Kubernetes 1.16+ cluster, you must: +To use Auto Deploy on a Kubernetes 1.16+ cluster: -1. Set the following in the [`.gitlab/auto-deploy-values.yaml` file](customize.md#customize-values-for-helm-chart): +1. If you are deploying your application for the first time on GitLab 13.0 or + newer, no configuration should be required. - ```yml +1. On GitLab 12.10 or older, set the following in the [`.gitlab/auto-deploy-values.yaml` file](customize.md#customize-values-for-helm-chart): + + ```yaml deploymentApiVersion: apps/v1 ``` -1. Set the: - - - `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to `2`. - - `POSTGRES_VERSION` variable to `9.6.16` or higher. +1. If you have an in-cluster PostgreSQL database installed with + `AUTO_DEVOPS_POSTGRES_CHANNEL` set to `1`, follow the [guide to upgrade + PostgreSQL](upgrading_postgresql.md). - This will opt-in to using a version of the PostgreSQL chart that supports Kubernetes - 1.16 and higher. +1. If you are deploying your application for the first time and are using + GitLab 12.9 or 12.10, set `AUTO_DEVOPS_POSTGRES_CHANNEL` to `2`. -DANGER: **Danger:** Opting into `AUTO_DEVOPS_POSTGRES_CHANNEL` version -`2` will delete the version `1` PostgreSQL database. Please follow the -guide on [upgrading PostgreSQL](upgrading_postgresql.md) to backup and -restore your database before opting into version `2`. +DANGER: **Danger:** On GitLab 12.9 and 12.10, opting into +`AUTO_DEVOPS_POSTGRES_CHANNEL` version `2` deletes the version `1` PostgreSQL +database. Follow the [guide to upgrading PostgreSQL](upgrading_postgresql.md) +to back up and restore your database before opting into version `2` (On +GitLab 13.0, an additional variable is required to trigger the database +deletion). ### Migrations > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21955) in GitLab 11.4 -Database initialization and migrations for PostgreSQL can be configured to run +You can configure database initialization and migrations for PostgreSQL to run within the application pod by setting the project variables `DB_INITIALIZE` and `DB_MIGRATE` respectively. -If present, `DB_INITIALIZE` will be run as a shell command within an -application pod as a Helm post-install hook. As some applications will -not run without a successful database initialization step, GitLab will -deploy the first release without the application deployment and only the -database initialization step. After the database initialization completes, -GitLab will deploy a second release with the application deployment as -normal. +If present, `DB_INITIALIZE` is run as a shell command within an application pod +as a Helm post-install hook. As some applications can't run without a successful +database initialization step, GitLab deploys the first release without the +application deployment, and only the database initialization step. After the database +initialization completes, GitLab deploys a second release with the application +deployment as normal. Note that a post-install hook means that if any deploy succeeds, -`DB_INITIALIZE` will not be processed thereafter. +`DB_INITIALIZE` won't be processed thereafter. -If present, `DB_MIGRATE` will be run as a shell command within an application pod as +If present, `DB_MIGRATE` is run as a shell command within an application pod as a Helm pre-upgrade hook. For example, in a Rails application in an image built with @@ -362,38 +386,39 @@ For example, in a Rails application in an image built with - `DB_INITIALIZE` can be set to `RAILS_ENV=production /bin/herokuish procfile exec bin/rails db:setup` - `DB_MIGRATE` can be set to `RAILS_ENV=production /bin/herokuish procfile exec bin/rails db:migrate` -Unless you have a `Dockerfile` in your repo, your image is built with -Herokuish, and you must prefix commands run in these images with `/bin/herokuish -procfile exec` to replicate the environment where your application will run. +Unless your repository contains a `Dockerfile`, your image is built with +Herokuish, and you must prefix commands run in these images with +`/bin/herokuish procfile exec` to replicate the environment where your application +will run. ### Workers -Some web applications need to run extra deployments for "worker processes". For -example, it is common in a Rails application to have a separate worker process +Some web applications must run extra deployments for "worker processes". For +example, Rails applications commonly use separate worker processes to run background tasks like sending emails. The [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app) -used in Auto Deploy [has support for running worker -processes](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/9). +used in Auto Deploy +[has support for running worker processes](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/9). -In order to run a worker, you'll need to ensure that it is able to respond to +To run a worker, you must ensure the worker can respond to the standard health checks, which expect a successful HTTP response on port -`5000`. For [Sidekiq](https://github.com/mperham/sidekiq), you could make use of -the [`sidekiq_alive` gem](https://rubygems.org/gems/sidekiq_alive) to do this. +`5000`. For [Sidekiq](https://github.com/mperham/sidekiq), you can use +the [`sidekiq_alive` gem](https://rubygems.org/gems/sidekiq_alive). -In order to work with Sidekiq, you'll also need to ensure your deployments have -access to a Redis instance. Auto DevOps won't deploy this for you so you'll -need to: +To work with Sidekiq, you must also ensure your deployments have +access to a Redis instance. Auto DevOps won't deploy this instance for you, so +you must: - Maintain your own Redis instance. -- Set a CI variable `K8S_SECRET_REDIS_URL`, which the URL of this instance to - ensure it's passed into your deployments. +- Set a CI variable `K8S_SECRET_REDIS_URL`, which is the URL of this instance, + to ensure it's passed into your deployments. -Once you have configured your worker to respond to health checks, run a Sidekiq +After configuring your worker to respond to health checks, run a Sidekiq worker for your Rails application. You can enable workers by setting the following in the [`.gitlab/auto-deploy-values.yaml` file](customize.md#customize-values-for-helm-chart): -```yml +```yaml workers: sidekiq: replicaCount: 1 @@ -417,7 +442,7 @@ workers: By default, all Kubernetes pods are [non-isolated](https://kubernetes.io/docs/concepts/services-networking/network-policies/#isolated-and-non-isolated-pods), -meaning that they will accept traffic to and from any source. You can use +and accept traffic to and from any source. You can use [NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) to restrict connections to and from selected pods, namespaces, and the Internet. @@ -437,13 +462,13 @@ networkPolicy: enabled: true ``` -The default policy deployed by the auto deploy pipeline will allow -traffic within a local namespace and from the `gitlab-managed-apps` -namespace. All other inbound connection will be blocked. Outbound +The default policy deployed by the Auto Deploy pipeline allows +traffic within a local namespace, and from the `gitlab-managed-apps` +namespace. All other inbound connections are blocked. Outbound traffic (for example, to the Internet) is not affected by the default policy. You can also provide a custom [policy specification](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#networkpolicyspec-v1-networking-k8s-io) -via the `.gitlab/auto-deploy-values.yaml` file, for example: +in the `.gitlab/auto-deploy-values.yaml` file, for example: ```yaml networkPolicy: @@ -461,16 +486,19 @@ networkPolicy: app.gitlab.com/managed_by: gitlab ``` -For more information on how to install Network Policies, see +For more information on installing Network Policies, see [Install Cilium using GitLab CI/CD](../../user/clusters/applications.md#install-cilium-using-gitlab-cicd). ### Web Application Firewall (ModSecurity) customization > [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/44) in GitLab 12.8. -Customization on an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) or on a deployment base is available for clusters with [ModSecurity installed](../../user/clusters/applications.md#web-application-firewall-modsecurity). +Customization on an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) +or on a deployment base is available for clusters with +[ModSecurity installed](../../user/clusters/applications.md#web-application-firewall-modsecurity). -To enable ModSecurity with Auto Deploy, you need to create a `.gitlab/auto-deploy-values.yaml` file in your project with the following attributes. +To enable ModSecurity with Auto Deploy, you must create a `.gitlab/auto-deploy-values.yaml` +file in your project with the following attributes. |Attribute | Description | Default | -----------|-------------|---------| @@ -481,7 +509,7 @@ To enable ModSecurity with Auto Deploy, you need to create a `.gitlab/auto-deplo In the following `auto-deploy-values.yaml` example, some custom settings are enabled for ModSecurity. Those include setting its engine to process rules instead of only logging them, while adding two specific -rules which are header-based: +header-based rules: ```yaml ingress: @@ -500,17 +528,17 @@ ingress: ### Running commands in the container Applications built with [Auto Build](#auto-build) using Herokuish, the default -unless you have [a custom Dockerfile](#auto-build-using-a-dockerfile), may require -commands to be wrapped as follows: +unless your repository contains [a custom Dockerfile](#auto-build-using-a-dockerfile), +may require commands to be wrapped as follows: ```shell /bin/herokuish procfile exec $COMMAND ``` -This might be necessary, for example, when: +Some of the reasons you may need to wrap commands: - Attaching using `kubectl exec`. -- Using GitLab's [Web Terminal](../../ci/environments.md#web-terminals). +- Using GitLab's [Web Terminal](../../ci/environments/index.md#web-terminals). For example, to start a Rails console from the application root directory, run: @@ -520,12 +548,12 @@ For example, to start a Rails console from the application root directory, run: ## Auto Monitoring -Once your application is deployed, Auto Monitoring makes it possible to monitor +After your application deploys, Auto Monitoring helps you monitor your application's server and response metrics right out of the box. Auto Monitoring uses [Prometheus](../../user/project/integrations/prometheus.md) to -get system metrics such as CPU and memory usage directly from +retrieve system metrics, such as CPU and memory usage, directly from [Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md), -and response metrics such as HTTP error rates, latency, and throughput from the +and response metrics, such as HTTP error rates, latency, and throughput, from the [NGINX server](../../user/project/integrations/prometheus_library/nginx_ingress.md). The metrics include: @@ -538,14 +566,14 @@ GitLab provides some initial alerts for you after you install Prometheus: - Ingress status code `500` > 0.1% - NGINX status code `500` > 0.1% -To make use of Auto Monitoring: +To use Auto Monitoring: 1. [Install and configure the requirements](index.md#requirements). -1. [Enable Auto DevOps](index.md#enablingdisabling-auto-devops) if you haven't done already. -1. Finally, go to your project's **CI/CD > Pipelines** and run a pipeline. -1. Once the pipeline finishes successfully, open the - [monitoring dashboard for a deployed environment](../../ci/environments.md#monitoring-environments) +1. [Enable Auto DevOps](index.md#enablingdisabling-auto-devops), if you haven't done already. +1. Navigate to your project's **{rocket}** **CI/CD > Pipelines** and click **Run Pipeline**. +1. After the pipeline finishes successfully, open the + [monitoring dashboard for a deployed environment](../../ci/environments/index.md#monitoring-environments) to view the metrics of your deployed application. To view the metrics of the - whole Kubernetes cluster, navigate to **Operations > Metrics**. + whole Kubernetes cluster, navigate to **{cloud-gear}** **Operations > Metrics**.  diff --git a/doc/topics/autodevops/upgrading_postgresql.md b/doc/topics/autodevops/upgrading_postgresql.md index ccb009905eb..893f7ba7cde 100644 --- a/doc/topics/autodevops/upgrading_postgresql.md +++ b/doc/topics/autodevops/upgrading_postgresql.md @@ -39,7 +39,7 @@ being modified after the database dump is created. 1. Get the Kubernetes namespace for the environment. It typically looks like `<project-name>-<project-id>-<environment>`. In our example, the namespace is called `minimal-ruby-app-4349298-production`. - ```sh + ```shell $ kubectl get ns NAME STATUS AGE @@ -48,13 +48,13 @@ being modified after the database dump is created. 1. For ease of use, export the namespace name: - ```sh + ```shell export APP_NAMESPACE=minimal-ruby-app-4349298-production ``` 1. Get the deployment name for your application with the following command. In our example, the deployment name is `production`. - ```sh + ```shell $ kubectl get deployment --namespace "$APP_NAMESPACE" NAME READY UP-TO-DATE AVAILABLE AGE production 2/2 2 2 7d21h @@ -64,7 +64,7 @@ being modified after the database dump is created. 1. To prevent the database from being modified, set replicas to 0 for the deployment with the following command. We use the deployment name from the previous step (`deployments/<DEPLOYMENT_NAME>`). - ```sh + ```shell $ kubectl scale --replicas=0 deployments/production --namespace "$APP_NAMESPACE" deployment.extensions/production scaled ``` @@ -75,7 +75,7 @@ being modified after the database dump is created. 1. Get the service name for PostgreSQL. The name of the service should end with `-postgres`. In our example the service name is `production-postgres`. - ```sh + ```shell $ kubectl get svc --namespace "$APP_NAMESPACE" NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE production-auto-deploy ClusterIP 10.30.13.90 <none> 5000/TCP 7d14h @@ -84,7 +84,7 @@ being modified after the database dump is created. 1. Get the pod name for PostgreSQL with the following command. In our example, the pod name is `production-postgres-5db86568d7-qxlxv`. - ```sh + ```shell $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=production-postgres NAME READY STATUS RESTARTS AGE production-postgres-5db86568d7-qxlxv 1/1 Running 0 7d14h @@ -92,7 +92,7 @@ being modified after the database dump is created. 1. Connect to the pod with: - ```sh + ```shell kubectl exec -it production-postgres-5db86568d7-qxlxv --namespace "$APP_NAMESPACE" bash ``` @@ -104,7 +104,7 @@ being modified after the database dump is created. - You will be asked for the database password, the default is `testing-password`. - ```sh + ```shell ## Format is: # pg_dump -h SERVICE_NAME -U USERNAME DATABASE_NAME > /tmp/backup.sql @@ -115,7 +115,7 @@ being modified after the database dump is created. 1. Download the dump file with the following command: - ```sh + ```shell kubectl cp --namespace "$APP_NAMESPACE" production-postgres-5db86568d7-qxlxv:/tmp/backup.sql backup.sql ``` @@ -126,12 +126,12 @@ volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) used to store the underlying data for PostgreSQL is marked as `Delete` when the pods and pod claims that use the volume is deleted. -This is signficant as, when you opt into the newer 8.2.1 PostgreSQL, the older 0.7.1 PostgreSQL is +This is significant as, when you opt into the newer 8.2.1 PostgreSQL, the older 0.7.1 PostgreSQL is deleted causing the persistent volumes to be deleted as well. You can verify this by using the following command: -```sh +```shell $ kubectl get pv NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE pvc-0da80c08-5239-11ea-9c8d-42010a8e0096 8Gi RWO Delete Bound minimal-ruby-app-4349298-staging/staging-postgres standard 7d22h @@ -145,7 +145,7 @@ interested in keeping the volumes for the staging and production of the `minimal-ruby-app-4349298` application, the volume names here are `pvc-0da80c08-5239-11ea-9c8d-42010a8e0096` and `pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096`: -```sh +```shell $ kubectl patch pv pvc-0da80c08-5239-11ea-9c8d-42010a8e0096 -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' persistentvolume/pvc-0da80c08-5239-11ea-9c8d-42010a8e0096 patched $ kubectl patch pv pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096 -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' @@ -164,17 +164,19 @@ deleted, you can choose to retain the [persistent volume](#retain-persistent-volumes). TIP: **Tip:** You can also -[scope](../../ci/environments.md#scoping-environments-with-specs) the -`AUTO_DEVOPS_POSTGRES_CHANNEL` and `POSTGRES_VERSION` variables to -specific environments, e.g. `staging`. +[scope](../../ci/environments/index.md#scoping-environments-with-specs) the +`AUTO_DEVOPS_POSTGRES_CHANNEL`, `AUTO_DEVOPS_POSTGRES_DELETE_V1` and +`POSTGRES_VERSION` variables to specific environments, e.g. `staging`. 1. Set `AUTO_DEVOPS_POSTGRES_CHANNEL` to `2`. This opts into using the newer 8.2.1-based PostgreSQL, and removes the older 0.7.1-based PostgreSQL. -1. Set `POSTGRES_VERSION` to `9.6.16`. This is the minimum PostgreSQL +1. Set `AUTO_DEVOPS_POSTGRES_DELETE_V1` to a non-empty value. This flag is a + safeguard to prevent accidental deletion of databases. +1. Set `POSTGRES_VERSION` to `11.7`. This is the minimum PostgreSQL version supported. 1. Set `PRODUCTION_REPLICAS` to `0`. For other environments, use - `REPLICAS` with an [environment scope](../../ci/environments.md#scoping-environments-with-specs). + `REPLICAS` with an [environment scope](../../ci/environments/index.md#scoping-environments-with-specs). 1. If you have set the `DB_INITIALIZE` or `DB_MIGRATE` variables, either remove the variables, or rename the variables temporarily to `XDB_INITIALIZE` or the `XDB_MIGRATE` to effectively disable them. @@ -190,7 +192,7 @@ specific environments, e.g. `staging`. 1. Get the pod name for the new PostgreSQL, in our example, the pod name is `production-postgresql-0`: - ```sh + ```shell $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=postgresql NAME READY STATUS RESTARTS AGE production-postgresql-0 1/1 Running 0 19m @@ -198,13 +200,13 @@ specific environments, e.g. `staging`. 1. Copy the dump file from the backup steps to the pod: - ```sh + ```shell kubectl cp --namespace "$APP_NAMESPACE" backup.sql production-postgresql-0:/tmp/backup.sql ``` 1. Connect to the pod: - ```sh + ```shell kubectl exec -it production-postgresql-0 --namespace "$APP_NAMESPACE" bash ``` @@ -214,7 +216,7 @@ specific environments, e.g. `staging`. - `USERNAME` is the username you have configured for PostgreSQL. The default is `user`. - `DATABASE_NAME` is usually the environment name. - ```sh + ```shell ## Format is: # psql -U USERNAME -d DATABASE_NAME < /tmp/backup.sql diff --git a/doc/topics/git/how_to_install_git/index.md b/doc/topics/git/how_to_install_git/index.md index 2c3cd61f050..75ea6183a32 100644 --- a/doc/topics/git/how_to_install_git/index.md +++ b/doc/topics/git/how_to_install_git/index.md @@ -6,7 +6,7 @@ article_type: user guide date: 2017-05-15 description: 'This article describes how to install Git on macOS, Ubuntu Linux and Windows.' type: howto -last_updated: 2019-05-31 +last_updated: 2020-04-22 --- # Installing Git @@ -32,34 +32,47 @@ use one of the aforementioned methods. ### Installing Xcode -Xcode is needed by Homebrew to build dependencies. -You can install [XCode](https://developer.apple.com/xcode/) -through the macOS App Store. +To build dependencies, Homebrew needs the XCode Command Line Tools. Install +it by running in your terminal: + +```shell +xcode-select --install +``` + +Click **Install** to download and install it. Alternatively, you can install +the entire [XCode](https://developer.apple.com/xcode/) package through the +macOS App Store. ### Installing Homebrew -Once Xcode is installed browse to the [Homebrew website](https://brew.sh/index.html) +With Xcode installed, browse to the [Homebrew website](https://brew.sh/index.html) for the official Homebrew installation instructions. ### Installing Git via Homebrew -With Homebrew installed you are now ready to install Git. -Open a Terminal and enter in the following command: +With Homebrew installed, you are now ready to install Git. +Open a terminal and enter the following command: ```shell brew install git ``` -Congratulations you should now have Git installed via Homebrew. +Congratulations! You should now have Git installed via Homebrew. + +To verify that Git works on your system, run: + +```shell +git --version +``` -Next read our article on [adding an SSH key to GitLab](../../../ssh/README.md). +Next, read our article on [adding an SSH key to GitLab](../../../ssh/README.md). ## Install Git on Ubuntu Linux On Ubuntu and other Linux operating systems -it is recommended to use the built in package manager to install Git. +it is recommended to use the built-in package manager to install Git. -Open a Terminal and enter in the following commands +Open a terminal and enter the following commands to install the latest Git from the official Git maintained package archives: ```shell @@ -68,15 +81,21 @@ sudo apt-get update sudo apt-get install git ``` -Congratulations you should now have Git installed via the Ubuntu package manager. +Congratulations! You should now have Git installed via the Ubuntu package manager. + +To verify that Git works on your system, run: + +```shell +git --version +``` -Next read our article on [adding an SSH key to GitLab](../../../ssh/README.md). +Next, read our article on [adding an SSH key to GitLab](../../../ssh/README.md). ## Installing Git on Windows from the Git website -Browse to the [Git website](https://git-scm.com/) and download and install Git for Windows. +Open the [Git website](https://git-scm.com/) and download and install Git for Windows. -Next read our article on [adding an SSH key to GitLab](../../../ssh/README.md). +Next, read our article on [adding an SSH key to GitLab](../../../ssh/README.md). <!-- ## Troubleshooting diff --git a/doc/topics/git/index.md b/doc/topics/git/index.md index 019194d1bba..9e6875312f3 100644 --- a/doc/topics/git/index.md +++ b/doc/topics/git/index.md @@ -27,21 +27,27 @@ The following resources will help you get started with Git: - [Git on the Server - GitLab](https://git-scm.com/book/en/v2/Git-on-the-Server-GitLab) - [How to install Git](how_to_install_git/index.md) - [Start using Git on the command line](../../gitlab-basics/start-using-git.md) -- [Command Line basic commands](../../gitlab-basics/command-line-commands.md) +- [Command line file editing basic commands](../../gitlab-basics/command-line-commands.md) - [GitLab Git Cheat Sheet (download)](https://about.gitlab.com/images/press/git-cheat-sheet.pdf) - Commits: - [Revert a commit](../../user/project/merge_requests/revert_changes.md#reverting-a-commit) - [Cherry-picking a commit](../../user/project/merge_requests/cherry_pick_changes.md#cherry-picking-a-commit) - [Squashing commits](../gitlab_flow.md#squashing-commits-with-rebase) + - [Squash-and-merge](../../user/project/merge_requests/squash_and_merge.md) + - [Signing commits](../../user/project/repository/gpg_signed_commits/index.md) +- [Git stash](../../university/training/topics/stash.md) +- [Git file blame](../../user/project/repository/git_blame.md) +- [Git file history](../../user/project/repository/git_history.md) +- [Git tags](../../university/training/user_training.md#tags) ### Concepts -The following are resources about version control concepts: +The following are resources on version control concepts: - [Git concepts](../../university/training/user_training.md#git-concepts) - [Why Git is Worth the Learning Curve](https://about.gitlab.com/blog/2017/05/17/learning-curve-is-the-biggest-challenge-developers-face-with-git/) - [The future of SaaS hosted Git repository pricing](https://about.gitlab.com/blog/2016/05/11/git-repository-pricing/) -- [Git website topic about version control](https://git-scm.com/book/en/v2/Getting-Started-About-Version-Control) +- [Git website on version control](https://git-scm.com/book/en/v2/Getting-Started-About-Version-Control) - [GitLab University presentation about Version Control](https://docs.google.com/presentation/d/16sX7hUrCZyOFbpvnrAFrg6tVO5_yT98IgdAqOmXwBho/edit?usp=sharing) ## Git tips @@ -61,9 +67,10 @@ If you have problems with Git, the following may help: ## Branching strategies +- [Feature branch workflow](../../gitlab-basics/feature_branch_workflow.md) +- [GitLab Flow](../gitlab_flow.md) - [Git Branching - Branches in a Nutshell](https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell) - [Git Branching - Branching Workflows](https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows) -- [GitLab Flow](https://about.gitlab.com/blog/2014/09/29/gitlab-flow/) ## Advanced use @@ -79,7 +86,7 @@ The following are advanced topics for those who want to get the most out of Git: [Gitignore templates](../../api/templates/gitignores.md) API allow for Git-related queries from GitLab. -## Git LFS +## Git Large File Storage (LFS) The following relate to Git Large File Storage: @@ -88,5 +95,5 @@ The following relate to Git Large File Storage: - [Removing objects from LFS](lfs/index.md#removing-objects-from-lfs) - [GitLab Git LFS user documentation](lfs/index.md) - [GitLab Git LFS admin documentation](../../administration/lfs/index.md) -- [git-annex to Git-LFS migration guide](lfs/migrate_from_git_annex_to_git_lfs.md) +- [Git Annex to Git LFS migration guide](lfs/migrate_from_git_annex_to_git_lfs.md) - [Towards a production quality open source Git LFS server](https://about.gitlab.com/blog/2015/08/13/towards-a-production-quality-open-source-git-lfs-server/) diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index 325a87215c4..33b7fa45691 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -2,7 +2,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs/index.html' --- -# Git LFS +# Git Large File Storage (LFS) Managing large files such as audio, video and graphics files has always been one of the shortcomings of Git. The general recommendation is to not have Git repositories @@ -154,7 +154,7 @@ also edit it: git lfs unlock images/banner.png ``` -You can also unlock by id: +You can also unlock by ID: ```shell git lfs unlock --id=123 @@ -217,10 +217,10 @@ If you push a LFS object to a project and you receive an error similar to: the LFS client is trying to reach GitLab through HTTPS. However, your GitLab instance is being served on HTTP. -This behaviour is caused by Git LFS using HTTPS connections by default when a +This behavior is caused by Git LFS using HTTPS connections by default when a `lfsurl` is not set in the Git config. -To prevent this from happening, set the lfs url in project Git config: +To prevent this from happening, set the lfs URL in project Git config: ```shell git config --add lfs.url "http://gitlab.example.com/group/project.git/info/lfs" @@ -265,7 +265,7 @@ If you are storing LFS files outside of GitLab you can disable LFS on the projec ### Hosting LFS objects externally -It is possible to host LFS objects externally by setting a custom LFS url with `git config -f .lfsconfig lfs.url https://example.com/<project>.git/info/lfs`. +It is possible to host LFS objects externally by setting a custom LFS URL with `git config -f .lfsconfig lfs.url https://example.com/<project>.git/info/lfs`. You might choose to do this if you are using an appliance like a Sonatype Nexus to store LFS data. If you choose to use an external LFS store, GitLab will not be able to verify LFS objects which means that pushes will fail if you have GitLab LFS support enabled. diff --git a/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md b/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md index d23199fa556..05b749d7b24 100644 --- a/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md @@ -1,16 +1,16 @@ # Migration guide from Git Annex to Git LFS >**Note:** -Git Annex support [has been removed][issue-remove-annex] in GitLab Enterprise +Git Annex support [has been removed](https://gitlab.com/gitlab-org/gitlab/issues/1648) in GitLab Enterprise Edition 9.0 (2017/03/22). -Both [Git Annex][] and [Git LFS][] are tools to manage large files in Git. +Both [Git Annex](http://git-annex.branchable.com/) and [Git LFS](https://git-lfs.github.com/) are tools to manage large files in Git. ## History -Git Annex [was introduced in GitLab Enterprise Edition 7.8][post-3], at a time +Git Annex [was introduced in GitLab Enterprise Edition 7.8](https://about.gitlab.com/blog/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/), at a time where Git LFS didn't yet exist. A few months later, GitLab brought support for -Git LFS in [GitLab 8.2][post-2] and is available for both Community and +Git LFS in [GitLab 8.2](https://about.gitlab.com/blog/2015/11/23/announcing-git-lfs-support-in-gitlab/) and is available for both Community and Enterprise editions. ## Differences between Git Annex and Git LFS @@ -71,13 +71,13 @@ Before changing anything, make sure you have a backup of your repository first. There are a couple of ways to do that, but you can simply clone it to another local path and maybe push it to GitLab if you want a remote backup as well. Here you'll find a guide on -[how to back up a **git-annex** repository to an external hard drive][bkp-ext-drive]. +[how to back up a **git-annex** repository to an external hard drive](https://www.thomas-krenn.com/en/wiki/Git-annex_Repository_on_an_External_Hard_Drive). Since Annex files are stored as objects with symlinks and cannot be directly modified, we need to first remove those symlinks. NOTE: **Note:** -Make sure the you read about the [`direct` mode][annex-direct] as it contains +Make sure the you read about the [`direct` mode](https://git-annex.branchable.com/direct_mode/) as it contains useful information that may fit in your use case. Note that `annex direct` is deprecated in Git Annex version 6, so you may need to upgrade your repository if the server also has Git Annex 6 installed. Read more in the @@ -114,7 +114,7 @@ if the server also has Git Annex 6 installed. Read more in the direct ok ``` -1. Disable Git Annex with [`annex uninit`][uninit]: +1. Disable Git Annex with [`annex uninit`](https://git-annex.branchable.com/git-annex-uninit/): ```shell git annex uninit @@ -170,7 +170,7 @@ GitLab.com), therefore, you don't need to do anything server-side. ``` If the terminal doesn't prompt you with a full response on `git-lfs` commands, - [install the Git LFS client][install-lfs] first. + [install the Git LFS client](https://git-lfs.github.com/) first. 1. Inside the repo, run the following command to initiate LFS: @@ -235,19 +235,8 @@ git annex uninit ## Further Reading -- (Blog Post) [Getting Started with Git FLS][post-1] -- (Blog Post) [Announcing LFS Support in GitLab][post-2] -- (Blog Post) [GitLab Annex Solves the Problem of Versioning Large Binaries with Git][post-3] +- (Blog Post) [Getting Started with Git FLS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) +- (Blog Post) [Announcing LFS Support in GitLab](https://about.gitlab.com/blog/2015/11/23/announcing-git-lfs-support-in-gitlab/) +- (Blog Post) [GitLab Annex Solves the Problem of Versioning Large Binaries with Git](https://about.gitlab.com/blog/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/) - (GitLab Docs) [Git Annex](../../../administration/git_annex.md) - (GitLab Docs) [Git LFS](index.md) - -[annex-direct]: https://git-annex.branchable.com/direct_mode/ -[bkp-ext-drive]: https://www.thomas-krenn.com/en/wiki/Git-annex_Repository_on_an_External_Hard_Drive -[Git Annex]: http://git-annex.branchable.com/ -[Git LFS]: https://git-lfs.github.com/ -[install-lfs]: https://git-lfs.github.com/ -[issue-remove-annex]: https://gitlab.com/gitlab-org/gitlab/issues/1648 -[post-1]: https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/ -[post-2]: https://about.gitlab.com/blog/2015/11/23/announcing-git-lfs-support-in-gitlab/ -[post-3]: https://about.gitlab.com/blog/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/ -[uninit]: https://git-annex.branchable.com/git-annex-uninit/ diff --git a/doc/topics/git/numerous_undo_possibilities_in_git/index.md b/doc/topics/git/numerous_undo_possibilities_in_git/index.md index 9d175b2edb1..8597325db7b 100644 --- a/doc/topics/git/numerous_undo_possibilities_in_git/index.md +++ b/doc/topics/git/numerous_undo_possibilities_in_git/index.md @@ -24,7 +24,7 @@ Git is really deleted](https://git-scm.com/book/en/v2/Git-Internals-Maintenance- This means that until Git automatically cleans detached commits (which cannot be accessed by branch or tag) it will be possible to view them with `git reflog` command -and access them with direct commit-id. Read more about _[redoing the undo](#redoing-the-undo)_ on the section below. +and access them with direct commit ID. Read more about _[redoing the undo](#redoing-the-undo)_ in the section below. ## Introduction @@ -36,12 +36,12 @@ a file into the **staged** state, which is then committed (`git commit`) to your local repository. After that, file can be shared with other developers (`git push`). Here's what we'll cover in this tutorial: -- [Undo local changes](#undo-local-changes) which were not pushed to remote repository: +- [Undo local changes](#undo-local-changes) which were not pushed to a remote repository: - Before you commit, in both unstaged and staged state. - After you committed. -- Undo changes after they are pushed to remote repository: +- Undo changes after they are pushed to a remote repository: - [Without history modification](#undo-remote-changes-without-changing-history) (preferred way). - [With history modification](#undo-remote-changes-with-modifying-history) (requires @@ -68,7 +68,7 @@ joined development of the same feature by multiple developers by default. When multiple developers develop the same feature on the same branch, clashing with every synchronization is unavoidable, but a proper or chosen Git Workflow will -prevent that anything is lost or out of sync when feature is complete. +prevent that anything is lost or out of sync when the feature is complete. You can also read through this blog post on [Git Tips & Tricks](https://about.gitlab.com/blog/2016/12/08/git-tips-and-tricks/) @@ -83,7 +83,7 @@ can be on various stages and each stage has a different approach on how to tackl ### Unstaged local changes (before you commit) When a change is made, but it is not added to the staged tree, Git itself -proposes a solution to discard changes to certain file. +proposes a solution to discard changes to a certain file. Suppose you edited a file to change the content using your favorite editor: @@ -233,7 +233,7 @@ last known good commit (we assume `A`) and first known bad commit (where bug was git bisect A..E ``` -Bisect will provide us with commit-id of the middle commit to test, and then guide us +Bisect will provide us with commit ID of the middle commit to test, and then guide us through simple bisection process. You can read more about it [in official Git Tools](https://git-scm.com/book/en/v2/Git-Tools-Debugging-with-Git) In our example we will end up with commit `B`, that introduced bug/error. We have 4 options on how to remove it (or part of it) from our repository. @@ -332,7 +332,7 @@ history](#how-modifying-history-is-done) Sometimes you realize that the changes you undid were useful and you want them back. Well because of first paragraph you are in luck. Command `git reflog` enables you to *recall* detached local commits by referencing or applying them -via commit-id. Although, do not expect to see really old commits in reflog, because +via commit ID. Although, do not expect to see really old commits in reflog, because Git regularly [cleans the commits which are *unreachable* by branches or tags](https://git-scm.com/book/en/v2/Git-Internals-Maintenance-and-Data-Recovery). To view repository history and to track older commits you can use below command: @@ -353,7 +353,7 @@ eb37e74 HEAD@{6}: rebase -i (pick): Commit C 6e43d59 HEAD@{16}: commit: Commit B ``` -Output of command shows repository history. In first column there is commit-id, +Output of command shows repository history. In first column there is commit ID, in following column, number next to `HEAD` indicates how many commits ago something was made, after that indicator of action that was made (commit, rebase, merge, ...) and then on end description of that action. @@ -393,7 +393,7 @@ passwords, SSH keys, etc. It is and should not be used to hide mistakes, as it will make it harder to debug in case there are some other bugs. The main reason for this is that you loose the real development progress. **Also keep in mind that, even with modified history, commits are just detached and can still be -accessed through commit-id** - at least until all repositories perform +accessed through commit ID** - at least until all repositories perform the cleanup of detached commits (happens automatically).  @@ -426,7 +426,7 @@ Never modify the commit history of `master` or shared branch. After you know what you want to modify (how far in history or how which range of old commits), use `git rebase -i commit-id`. This command will then display all the commits from -current version to chosen commit-id and allow modification, squashing, deletion +current version to chosen commit ID and allow modification, squashing, deletion of that commits. ```shell diff --git a/doc/topics/git/partial_clone.md b/doc/topics/git/partial_clone.md index a48e1c19f5c..46318a7f30d 100644 --- a/doc/topics/git/partial_clone.md +++ b/doc/topics/git/partial_clone.md @@ -9,22 +9,7 @@ is a performance optimization that "allows Git to function without having a complete copy of the repository. The goal of this work is to allow Git better handle extremely large repositories." -## Enabling partial clone - -> [Introduced](https://gitlab.com/gitlab-org/gitaly/issues/1553) in GitLab 12.4. - -To enable partial clone, use the [feature flags API](../../api/features.md). -For example: - -```shell -curl --data "value=true" --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/features/gitaly_upload_pack_filter -``` - -Alternatively, flip the switch and enable the feature flag: - -```ruby -Feature.enable(:gitaly_upload_pack_filter) -``` +Git 2.22.0 or later is required. ## Filter by file size @@ -82,7 +67,7 @@ reduce the size of your working copy. ```plaintext # Clone the repo excluding all files -$ git clone --filter=blob:none --sparse git@gitlab.com:gitlab-com/www-gitlab-com/git +$ git clone --filter=blob:none --sparse git@gitlab.com:gitlab-com/www-gitlab-com.git Cloning into 'www-gitlab-com'... remote: Enumerating objects: 678296, done. remote: Counting objects: 100% (678296/678296), done. @@ -135,7 +120,7 @@ enabled on the Git server: many applications, each in a different subdirectory in the root. Create a file `shiny-app/.filterspec` using the GitLab web interface: - ```.gitignore + ```plaintext # Only the paths listed in the file will be downloaded when performing a # partial clone using `--filter=sparse:oid=shiny-app/.gitfilterspec` @@ -153,7 +138,7 @@ enabled on the Git server: shared-component-b/ ``` -1. *Create a new Git repository and fetch.* Support for `--filter=sparse:oid` +1. **Create a new Git repository and fetch.** Support for `--filter=sparse:oid` using the clone command is incomplete, so we will emulate the clone command by hand, using `git init` and `git fetch`. Follow [issue tracking support for `--filter=sparse:oid`](https://gitlab.com/gitlab-org/git/issues/4) diff --git a/doc/topics/git/troubleshooting_git.md b/doc/topics/git/troubleshooting_git.md index f8d812d37e6..010a811bd33 100644 --- a/doc/topics/git/troubleshooting_git.md +++ b/doc/topics/git/troubleshooting_git.md @@ -13,7 +13,7 @@ with Git. 'Broken pipe' errors can occur when attempting to push to a remote repository. When pushing you will usually see: -```text +```plaintext Write failed: Broken pipe fatal: The remote end hung up unexpectedly ``` @@ -56,7 +56,7 @@ Configuring *both* the client and the server is unnecessary. - On UNIX, edit `~/.ssh/config` (create the file if it doesn’t exist) and add or edit: - ```text + ```plaintext Host your-gitlab-instance-url.com ServerAliveInterval 60 ServerAliveCountMax 5 @@ -68,7 +68,7 @@ Configuring *both* the client and the server is unnecessary. **To configure SSH on the server side**, edit `/etc/ssh/sshd_config` and add: -```text +```plaintext ClientAliveInterval 60 ClientAliveCountMax 5 ``` @@ -93,7 +93,7 @@ to >= 2.9 (see [Broken pipe when pushing to Git repository](https://stackoverflo Users may experience the following error when attempting to push or pull using Git over SSH: -```text +```plaintext Please make sure you have the correct access rights and the repository exists. ... @@ -103,7 +103,7 @@ fatal: Could not read from remote repository. or -```text +```plaintext ssh_exchange_identification: Connection closed by remote host fatal: The remote end hung up unexpectedly ``` @@ -117,7 +117,7 @@ beginning. The default value is `10`. Increase `MaxStartups` on the GitLab server by adding or modifying the value in `/etc/ssh/sshd_config`: -```text +```plaintext MaxStartups 100:30:200 ``` @@ -140,7 +140,7 @@ If pulling/pushing from/to your repository ends up taking more than 50 seconds, a timeout will be issued with a log of the number of operations performed and their respective timings, like the example below: -```text +```plaintext remote: Running checks for branch: master remote: Scanning for LFS objects... (153ms) remote: Calculating new repository size... (cancelled after 729ms) @@ -153,7 +153,7 @@ and provide GitLab with more information on how to improve the service. If the buffer size is lower than what is allowed in the request, the action will fail with an error similar to the one below: -```text +```plaintext error: RPC failed; curl 18 transfer closed with outstanding read data remaining fatal: The remote end hung up unexpectedly fatal: early EOF diff --git a/doc/topics/git/useful_git_commands.md b/doc/topics/git/useful_git_commands.md index 63defb24eaf..711567dbd06 100644 --- a/doc/topics/git/useful_git_commands.md +++ b/doc/topics/git/useful_git_commands.md @@ -116,7 +116,7 @@ git log <file> If you get this error message: -```text +```plaintext fatal: ambiguous argument <file_name>: unknown revision or path not in the working tree. Use '--' to separate paths from revisions, like this: ``` diff --git a/doc/topics/gitlab_flow.md b/doc/topics/gitlab_flow.md index fb47899c585..4738c5b23d7 100644 --- a/doc/topics/gitlab_flow.md +++ b/doc/topics/gitlab_flow.md @@ -253,7 +253,7 @@ If you need to utilize some code that was introduced in `master` after you creat If your feature branch has a merge conflict, creating a merge commit is a standard way of solving this. NOTE: **Note:** -Sometimes you can use .gitattributes to reduce merge conflicts. +Sometimes you can use `.gitattributes` to reduce merge conflicts. For example, you can set your changelog file to use the [union merge driver](https://git-scm.com/docs/gitattributes#gitattributes-union) so that multiple new entries don't conflict with each other. The last reason for creating merge commits is to keep long-running feature branches up-to-date with the latest state of the project. diff --git a/doc/topics/web_application_firewall/img/guide_waf_ingress_disabled_settings_v12_10.png b/doc/topics/web_application_firewall/img/guide_waf_ingress_disabled_settings_v12_10.png Binary files differnew file mode 100644 index 00000000000..2dd6df3d37b --- /dev/null +++ b/doc/topics/web_application_firewall/img/guide_waf_ingress_disabled_settings_v12_10.png diff --git a/doc/university/support/README.md b/doc/university/support/README.md index a60107dc4e1..5e824bcb6f9 100644 --- a/doc/university/support/README.md +++ b/doc/university/support/README.md @@ -62,14 +62,14 @@ Sometimes we need to upgrade customers from old versions of GitLab to latest, so - Users - Groups - Projects - - [Backup using our Backup Rake task](../../raketasks/backup_restore.md#creating-a-backup-of-the-gitlab-system) + - [Back up using our backup Rake task](../../raketasks/backup_restore.md#back-up-gitlab) - [Upgrade to 5.0 source using our Upgrade documentation](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/4.2-to-5.0.md) - [Upgrade to 5.1 source](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/5.0-to-5.1.md) - [Upgrade to 6.0 source](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/5.1-to-6.0.md) - [Upgrade to 7.14 source](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/6.x-or-7.x-to-7.14.md) - [Perform the MySQL to PostgreSQL migration to convert your backup](../../update/mysql_to_postgresql.md) - [Upgrade to Omnibus 7.14](https://docs.gitlab.com/omnibus/update/README.html#upgrading-from-a-non-omnibus-installation-to-an-omnibus-installation) - - [Restore backup using our Restore Rake task](../../raketasks/backup_restore.md#restore) + - [Restore backup using our Restore Rake task](../../raketasks/backup_restore.md#restore-gitlab) - [Upgrade to latest EE](https://about.gitlab.com/update/) - (GitLab inc. only) Acquire and apply a license for the Enterprise Edition product, ask in #support - Perform a downgrade from [EE to CE](../../downgrade_ee_to_ce/README.md) @@ -103,7 +103,7 @@ Our integrations add great value to GitLab. User questions often relate to integ ### Learn about the Support process -Zendesk is our Support Centre and our main communication line with our Customers. We communicate with customers through several other channels too +Zendesk is our Support Center and our main communication line with our Customers. We communicate with customers through several other channels too - Familiarize yourself with ZenDesk: - [UI Overview](https://support.zendesk.com/hc/en-us/articles/203661806-Introduction-to-the-Zendesk-agent-interface) diff --git a/doc/university/training/end-user/README.md b/doc/university/training/end-user/README.md index 5fc65bb8a58..0465f48bb34 100644 --- a/doc/university/training/end-user/README.md +++ b/doc/university/training/end-user/README.md @@ -278,7 +278,7 @@ git rm '*.txt' git rm -r <dirname> ``` -If we want to remove a file from the repository but keep it on disk, say we forgot to add it to our .gitignore file then use `--cache`: +If we want to remove a file from the repository but keep it on disk, say we forgot to add it to our `.gitignore` file then use `--cache`: ```shell git rm <filename> --cache diff --git a/doc/university/training/topics/agile_git.md b/doc/university/training/topics/agile_git.md index 6434b710159..c5634bec27b 100644 --- a/doc/university/training/topics/agile_git.md +++ b/doc/university/training/topics/agile_git.md @@ -20,7 +20,7 @@ Branching in an Agile environment usually happens around user stories with one or more developers working on it. If more than one developer then another branch for each developer is also used -with their initials, and US id. +with their initials, and US ID. After its tested merge into master and remove the branch. diff --git a/doc/update/README.md b/doc/update/README.md index c7be3f3cb2b..8056460ceaa 100644 --- a/doc/update/README.md +++ b/doc/update/README.md @@ -13,7 +13,7 @@ Based on your installation, choose a section below that fits your needs. ## Omnibus Packages -- The [Omnibus update guide][omni-update] +- The [Omnibus update guide](https://docs.gitlab.com/omnibus/update/README.html) contains the steps needed to update an Omnibus GitLab package. ## Installation from source @@ -29,14 +29,14 @@ In the past we used separate documents for the upgrading instructions, but we have since switched to using a single document. The old upgrading guidelines can still be found in the Git repository: -- [Old upgrading guidelines for Community Edition][old-ce-upgrade-docs] -- [Old upgrading guidelines for Enterprise Edition][old-ee-upgrade-docs] +- [Old upgrading guidelines for Community Edition](https://gitlab.com/gitlab-org/gitlab-foss/tree/11-8-stable/doc/update) +- [Old upgrading guidelines for Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/tree/11-8-stable-ee/doc/update) ## Installation using Docker GitLab provides official Docker images for both Community and Enterprise editions. They are based on the Omnibus package and instructions on how to -update them are in [a separate document][omni-docker]. +update them are in [a separate document](https://docs.gitlab.com/omnibus/docker/README.html). ## Upgrading without downtime @@ -106,7 +106,7 @@ meet the other online upgrade requirements mentioned above. ### Steps -Steps to [upgrade without downtime][omni-zero-downtime]. +Steps to [upgrade without downtime](https://docs.gitlab.com/omnibus/update/README.html#zero-downtime-updates). ## Checking for background migrations before upgrading @@ -163,8 +163,8 @@ of the `background_migration` queue, [check for background migrations before upg ## Upgrading between editions -GitLab comes in two flavors: [Community Edition][ce] which is MIT licensed, -and [Enterprise Edition][ee] which builds on top of the Community Edition and +GitLab comes in two flavors: [Community Edition](https://about.gitlab.com/features/#community) which is MIT licensed, +and [Enterprise Edition](https://about.gitlab.com/features/#enterprise) which builds on top of the Community Edition and includes extra features mainly aimed at organizations with more than 100 users. Below you can find some guides to help you change editions easily. @@ -177,17 +177,17 @@ The following guides are for subscribers of the Enterprise Edition only. If you wish to upgrade your GitLab installation from Community to Enterprise Edition, follow the guides below based on the installation method: -- [Source CE to EE update guides][source-ce-to-ee] - The steps are very similar +- [Source CE to EE update guides](upgrading_from_ce_to_ee.md) - The steps are very similar to a version upgrade: stop the server, get the code, update config files for the new functionality, install libraries and do migrations, update the init script, start the application and check its status. -- [Omnibus CE to EE][omni-ce-ee] - Follow this guide to update your Omnibus +- [Omnibus CE to EE](https://docs.gitlab.com/omnibus/update/README.html#updating-community-edition-to-enterprise-edition) - Follow this guide to update your Omnibus GitLab Community Edition to the Enterprise Edition. ### Enterprise to Community Edition If you need to downgrade your Enterprise Edition installation back to Community -Edition, you can follow [this guide][ee-ce] to make the process as smooth as +Edition, you can follow [this guide](../downgrade_ee_to_ce/README.md) to make the process as smooth as possible. ## Version specific upgrading instructions @@ -227,15 +227,3 @@ for more information. - [Restoring from backup after a failed upgrade](restore_after_failure.md) - [Upgrading PostgreSQL Using Slony](upgrading_postgresql_using_slony.md), for upgrading a PostgreSQL database with minimal downtime. - -[omnidocker]: https://docs.gitlab.com/omnibus/docker/README.html -[old-ee-upgrade-docs]: https://gitlab.com/gitlab-org/gitlab/tree/11-8-stable-ee/doc/update -[old-ce-upgrade-docs]: https://gitlab.com/gitlab-org/gitlab-foss/tree/11-8-stable/doc/update -[source-ce-to-ee]: upgrading_from_ce_to_ee.md -[ee-ce]: ../downgrade_ee_to_ce/README.md -[ce]: https://about.gitlab.com/features/#community -[ee]: https://about.gitlab.com/features/#enterprise -[omni-ce-ee]: https://docs.gitlab.com/omnibus/update/README.html#updating-community-edition-to-enterprise-edition -[omni-docker]: https://docs.gitlab.com/omnibus/docker/README.html -[omni-update]: https://docs.gitlab.com/omnibus/update/README.html -[omni-zero-downtime]: https://docs.gitlab.com/omnibus/update/README.html#zero-downtime-updates diff --git a/doc/update/patch_versions.md b/doc/update/patch_versions.md index aaa464dfdcb..f226ad9ac28 100644 --- a/doc/update/patch_versions.md +++ b/doc/update/patch_versions.md @@ -12,7 +12,7 @@ You can select the tag in the version dropdown in the top left corner of GitLab ### 0. Backup -It's useful to make a backup just in case things go south. Depending on the installation method, backup commands vary, see the [backing up and restoring GitLab](../raketasks/backup_restore.md#creating-a-backup-of-the-gitlab-system) documentation. +It's useful to make a backup just in case things go south. Depending on the installation method, backup commands vary. See the [backing up and restoring GitLab](../raketasks/backup_restore.md) documentation. ### 1. Stop server diff --git a/doc/update/restore_after_failure.md b/doc/update/restore_after_failure.md index e01ae409bb3..c850bbff1cf 100644 --- a/doc/update/restore_after_failure.md +++ b/doc/update/restore_after_failure.md @@ -10,9 +10,9 @@ the previous version you were using. First, roll back the code or package. For source installations this involves checking out the older version (branch or tag). For Omnibus installations this -means installing the older .deb or .rpm package. Then, restore from a backup. +means installing the older `.deb` or `.rpm` package. Then, restore from a backup. Follow the instructions in the -[Backup and Restore](../raketasks/backup_restore.md#restore) +[Backup and Restore](../raketasks/backup_restore.md#restore-gitlab) documentation. ## Potential problems on the next upgrade diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md index 28d5fe7aa5f..2415d3b2741 100644 --- a/doc/update/upgrading_from_ce_to_ee.md +++ b/doc/update/upgrading_from_ce_to_ee.md @@ -7,7 +7,7 @@ comments: false NOTE: **NOTE** In the past we used separate documents for upgrading from Community Edition to Enterprise Edition. These documents can be found in the [`doc/update` directory of Enterprise Edition's source -code][old-ee-upgrade-docs]. +code](https://gitlab.com/gitlab-org/gitlab/tree/11-8-stable-ee/doc/update). If you want to upgrade the version only, for example 11.8 to 11.9, *without* changing the GitLab edition you are using (Community or Enterprise), see the @@ -133,5 +133,3 @@ Example: Additional instructions here. --> - -[old-ee-upgrade-docs]: https://gitlab.com/gitlab-org/gitlab/tree/11-8-stable-ee/doc/update diff --git a/doc/user/admin_area/abuse_reports.md b/doc/user/admin_area/abuse_reports.md index 36de817a29b..77f4a84cf6f 100644 --- a/doc/user/admin_area/abuse_reports.md +++ b/doc/user/admin_area/abuse_reports.md @@ -46,7 +46,7 @@ Blocking a user: The user will be notified with the [following message](https://gitlab.com/gitlab-org/gitlab/blob/master/app/workers/email_receiver_worker.rb#L38): -```text +```plaintext Your account has been blocked. If you believe this is in error, contact a staff member. ``` diff --git a/doc/user/admin_area/broadcast_messages.md b/doc/user/admin_area/broadcast_messages.md index ed03e8a57f3..5427b0c5200 100644 --- a/doc/user/admin_area/broadcast_messages.md +++ b/doc/user/admin_area/broadcast_messages.md @@ -20,10 +20,19 @@ You can style a message's content using the `a` and `br` HTML tags. The `br` tag ## Banners -Banners are shown on the top of a page. +Banners are shown on the top of a page and in Git remote responses.  +```bash +$ git push +... +remote: +remote: **Welcome** to GitLab :wave: +remote: +... +``` + ## Notifications Notifications are shown on the bottom right of a page and can contain placeholders. A placeholder is replaced with an attribute of the active user. Placeholders must be surrounded by curly braces, for example `{{name}}`. diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index 204573da02d..e63d0c7c882 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -136,19 +136,22 @@ you must provide the complete email address. #### Users statistics -The **Users statistics** page provides an overview of user accounts by role. Use this information -when validating seat usage of your subscription. +The **Users statistics** page provides an overview of user accounts by role, such as _Users with +highest role Maintainer_. -The page displays subtotals of all users matching criteria such as _Users with highest role -Maintainer_ and _Blocked users_. +The following totals are also included: -The **Total users** is calculated as: **Active users** + **Blocked users**. +- Active users +- Blocked users +- Total users -GitLab billing is based on the number of active users. For details of active users, see +GitLab billing is based on the number of **Active users**, calculated as **Total users** - +**Blocked users**. For details of active users, see [Choosing the number of users](../../subscriptions/index.md#choosing-the-number-of-users). -**Please note** that during the initial stage, the information won't be 100% accurate given that -background processes are still recollecting data. +NOTE: **Note:** +Users statistics are calculated daily, so user changes made since the last update won't be +reflected in the statistics. ### Administering Groups diff --git a/doc/user/admin_area/license.md b/doc/user/admin_area/license.md index fd74b2f311f..59b71b05b16 100644 --- a/doc/user/admin_area/license.md +++ b/doc/user/admin_area/license.md @@ -10,8 +10,8 @@ by **signing into your GitLab instance as an admin** or add it at installation time. The license has the form of a base64 encoded ASCII text with a `.gitlab-license` -extension and can be obtained when you [purchase one][pricing] or when you sign -up for a [free trial]. +extension and can be obtained when you [purchase one](https://about.gitlab.com/pricing/) or when you sign +up for a [free trial](https://about.gitlab.com/free-trial/). NOTE: **Note:** As of GitLab Enterprise Edition 9.4.0, a newly-installed instance without an @@ -104,9 +104,6 @@ expired license(s). It's possible to upload and view more than one license, but only the latest license will be used as the active license. -[free trial]: https://about.gitlab.com/free-trial/ -[pricing]: https://about.gitlab.com/pricing/ - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/admin_area/monitoring/health_check.md b/doc/user/admin_area/monitoring/health_check.md index f69584fcb94..91e29118e3e 100644 --- a/doc/user/admin_area/monitoring/health_check.md +++ b/doc/user/admin_area/monitoring/health_check.md @@ -4,8 +4,8 @@ type: concepts, howto # Health Check **(CORE ONLY)** -> - Liveness and readiness probes were [introduced][ce-10416] in GitLab 9.1. -> - The `health_check` endpoint was [introduced][ce-3888] in GitLab 8.8 and was +> - Liveness and readiness probes were [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10416) in GitLab 9.1. +> - The `health_check` endpoint was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3888) in GitLab 8.8 and was > deprecated in GitLab 9.1. > - [Access token](#access-token-deprecated) has been deprecated in GitLab 9.4 > in favor of [IP whitelist](#ip-whitelist). @@ -25,15 +25,15 @@ For details, see [how to add IPs to a whitelist for the monitoring endpoints](.. With default whitelist settings, the probes can be accessed from localhost using the following URLs: -```text +```plaintext GET http://localhost/-/health ``` -```text +```plaintext GET http://localhost/-/readiness ``` -```text +```plaintext GET http://localhost/-/liveness ``` @@ -45,7 +45,7 @@ are running. This endpoint circumvents Rails Controllers and is implemented as additional middleware `BasicHealthCheck` very early into the request processing lifecycle. -```text +```plaintext GET /-/health ``` @@ -57,7 +57,7 @@ curl 'https://gitlab.example.com/-/health' Example response: -```text +```plaintext GitLab OK ``` @@ -71,7 +71,7 @@ If the `all=1` parameter is specified, the check will also validate the dependent services (Database, Redis, Gitaly etc.) and gives a status for each. -```text +```plaintext GET /-/readiness GET /-/readiness?all=1 ``` @@ -111,7 +111,7 @@ Checks whether the application server is running. This probe is used to know if Rails Controllers are not deadlocked due to a multi-threading. -```text +```plaintext GET /-/liveness ``` @@ -148,7 +148,7 @@ accepted token can be found under the **Admin Area > Monitoring > Health check** The access token can be passed as a URL parameter: -```text +```plaintext https://gitlab.example.com/-/readiness?token=ACCESS_TOKEN ``` @@ -163,9 +163,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -[ce-10416]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10416 -[ce-3888]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3888 -[pingdom]: https://www.pingdom.com -[nagios-health]: https://nagios-plugins.org/doc/man/check_http.html -[newrelic-health]: https://docs.newrelic.com/docs/alerts/alert-policies/downtime-alerts/availability-monitoring diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index f05ef18da8e..83b29597bec 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -44,7 +44,7 @@ For instance, consider the following workflow: 1. Your team develops apps which require large files to be stored in the application repository. -1. Although you have enabled [Git LFS](../../../topics/git/lfs/index.md#git-lfs) +1. Although you have enabled [Git LFS](../../../topics/git/lfs/index.md#git-large-file-storage-lfs) to your project, your storage has grown significantly. 1. Before you exceed available storage, you set up a limit of 10 GB per repository. diff --git a/doc/user/admin_area/settings/email.md b/doc/user/admin_area/settings/email.md index 4fab560e081..a99897657ae 100644 --- a/doc/user/admin_area/settings/email.md +++ b/doc/user/admin_area/settings/email.md @@ -12,7 +12,7 @@ The logo in the header of some emails can be customized, see the [logo customiza ## Custom additional text **(PREMIUM ONLY)** -> [Introduced][ee-5031] in [GitLab Premium][eep] 10.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5031) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.7. The additional text will appear at the bottom of any email and can be used for legal/auditing/compliance reasons. @@ -22,9 +22,6 @@ legal/auditing/compliance reasons. 1. Enter your text in the **Additional text** field. 1. Click **Save**. -[ee-5031]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5031 -[eep]: https://about.gitlab.com/pricing/ - ## Custom hostname for private commit emails > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22560) in GitLab 11.5. diff --git a/doc/user/admin_area/settings/external_authorization.md b/doc/user/admin_area/settings/external_authorization.md index abf3d79b9ed..3a03e46fa1f 100644 --- a/doc/user/admin_area/settings/external_authorization.md +++ b/doc/user/admin_area/settings/external_authorization.md @@ -33,7 +33,7 @@ authorization service. Whenever access is granted or denied this is logged in a logfile called `external-policy-access-control.log`. -Read more about logs GitLab keeps in the [omnibus documentation][omnibus-log-docs]. +Read more about logs GitLab keeps in the [omnibus documentation](https://docs.gitlab.com/omnibus/settings/logs.html). ## Configuration @@ -62,7 +62,7 @@ The available required properties are: When using TLS Authentication with a self signed certificate, the CA certificate needs to be trusted by the openssl installation. When using GitLab installed using Omnibus, learn to install a custom CA in the -[omnibus documentation][omnibus-ssl-docs]. Alternatively learn where to install +[omnibus documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). Alternatively learn where to install custom certificates using `openssl version -d`. ## How it works @@ -127,6 +127,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -[omnibus-ssl-docs]: https://docs.gitlab.com/omnibus/settings/ssl.html -[omnibus-log-docs]: https://docs.gitlab.com/omnibus/settings/logs.html diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index f0e7bf272a7..761d640c535 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -25,7 +25,7 @@ Access the default page for admin area settings by navigating to | [Terms of Service and Privacy Policy](terms.md) | Include a Terms of Service agreement and Privacy Policy that all users must accept. | | [External Authentication](external_authorization.md#configuration) | External Classification Policy Authorization | | [Web terminal](../../../administration/integration/terminal.md#limiting-websocket-connection-time) | Set max session time for web terminal. | -| [Web IDE](../../project/web_ide/index.md#enabling-client-side-evaluation) | Manage Web IDE Features. | +| [Web IDE](../../project/web_ide/index.md#enabling-live-preview) | Manage Web IDE Features. | ## Integrations @@ -35,7 +35,7 @@ Access the default page for admin area settings by navigating to | [PlantUML](../../../administration/integration/plantuml.md#gitlab) | Allow rendering of PlantUML diagrams in Asciidoc documents. | | [Slack application](../../../user/project/integrations/gitlab_slack_application.md#configuration) **(FREE ONLY)** | Slack integration allows you to interact with GitLab via slash commands in a chat window. This option is only available on GitLab.com, though it may be [available for self-managed instances in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). | | [Third party offers](third_party_offers.md) | Control the display of third party offers. | -| [Snowplow](../../../telemetry/index.md#enabling-tracking) | Configure the Snowplow integration. | +| [Snowplow](../../../development/telemetry/snowplow.md) | Configure the Snowplow integration. | | [Google GKE](../../project/clusters/add_gke_clusters.md) | Google GKE integration allows you to provision GKE clusters from GitLab. | | [Amazon EKS](../../project/clusters/add_eks_clusters.md) | Amazon EKS integration allows you to provision EKS clusters from GitLab. | @@ -44,7 +44,7 @@ Access the default page for admin area settings by navigating to | Option | Description | | ------ | ----------- | | [Repository mirror](visibility_and_access_controls.md#allow-mirrors-to-be-set-up-for-projects) | Configure repository mirroring. | -| [Repository storage](../../../administration/repository_storage_types.md#how-to-migrate-to-hashed-storage) | Configure storage path settings. | +| [Repository storage](../../../administration/repository_storage_types.md) | Configure storage path settings. | | Repository maintenance | ([Repository checks](../../../administration/repository_checks.md) and [Housekeeping](../../../administration/housekeeping.md)). Configure automatic Git checks and housekeeping on repositories. | | [Repository static objects](../../../administration/static_objects_external_storage.md) | Serve repository static objects (for example, archives, blobs, ...) from an external storage (for example, a CDN). | @@ -74,7 +74,6 @@ Access the default page for admin area settings by navigating to | Option | Description | | ------ | ----------- | -| [Metrics - Influx](../../../administration/monitoring/performance/gitlab_configuration.md) | Enable and configure InfluxDB metrics. | | [Metrics - Prometheus](../../../administration/monitoring/prometheus/gitlab_metrics.md) | Enable and configure Prometheus metrics. | | [Metrics - Grafana](../../../administration/monitoring/performance/grafana_configuration.md#integration-with-gitlab-ui) | Enable and configure Grafana. | | [Profiling - Performance bar](../../../administration/monitoring/performance/performance_bar.md#enable-the-performance-bar-via-the-admin-panel) | Enable access to the Performance Bar for a given group. | @@ -103,7 +102,7 @@ Access the default page for admin area settings by navigating to | Option | Description | | ------ | ----------- | | [Email](email.md) | Various email settings. | -| [Help page](../../../customization/help_message.md) | Help page text and support page url. | +| [Help page](../../../customization/help_message.md) | Help page text and support page URL. | | [Pages](../../../administration/pages/index.md#custom-domain-verification) | Size and domain settings for static websites | | [Real-time features](../../../administration/polling.md) | Change this value to influence how frequently the GitLab UI polls for updates. | | [Gitaly timeouts](gitaly_timeouts.md) | Configure Gitaly timeouts. | diff --git a/doc/user/admin_area/settings/instance_template_repository.md b/doc/user/admin_area/settings/instance_template_repository.md index b297d48cb0a..bef4e31259f 100644 --- a/doc/user/admin_area/settings/instance_template_repository.md +++ b/doc/user/admin_area/settings/instance_template_repository.md @@ -40,7 +40,7 @@ are supported: Each template must go in its respective subdirectory, have the correct extension and not be empty. So, the hierarchy should look like this: -```text +```plaintext |-- README.md |-- Dockerfile |-- custom_dockerfile.dockerfile diff --git a/doc/user/admin_area/settings/protected_paths.md b/doc/user/admin_area/settings/protected_paths.md index 56f99d3e725..0cfaf5843d0 100644 --- a/doc/user/admin_area/settings/protected_paths.md +++ b/doc/user/admin_area/settings/protected_paths.md @@ -54,24 +54,3 @@ customized on **Admin > Network > Protected Paths**, along with these options:  Requests over the rate limit are logged into `auth.log`. - -## Migrate settings from GitLab 12.3 and earlier - -Omnibus GitLab protected paths throttle is deprecated and is scheduled for removal in -GitLab 13.0. Please see the [GitLab issue](https://gitlab.com/gitlab-org/gitlab/issues/29952) and the [Omnibus GitLab issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4688) for more information. - -NOTE: **Note:** If Omnibus settings are present, applications settings will be automatically ignored to avoid generating multiple requests blocks. - -To migrate from Omnibus GitLab 12.3 and earlier settings: - -1. Customize and enable your protected paths settings by following [Configure using GitLab UI](#configure-using-gitlab-ui) section. - -1. SSH into your frontend nodes and add to `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['rack_attack_admin_area_protected_paths_enabled'] = true - ``` - -1. [Reconfigure GitLab](../../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -That's it. Protected paths throttle are now managed by GitLab admin settings. diff --git a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md index 96a20681b2f..dc23865e730 100644 --- a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md @@ -4,7 +4,7 @@ type: reference # Rate limits on issue creation -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/55241) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28129) in GitLab 12.10. This setting allows you to rate limit the requests to the issue creation endpoint. It defaults to 300 requests per minute. diff --git a/doc/user/admin_area/settings/sign_up_restrictions.md b/doc/user/admin_area/settings/sign_up_restrictions.md index a7a6b0ecefd..8ef5ac8dc8f 100644 --- a/doc/user/admin_area/settings/sign_up_restrictions.md +++ b/doc/user/admin_area/settings/sign_up_restrictions.md @@ -50,14 +50,14 @@ the minimum number of characters a user must have in their password using the Gi ## Whitelist email domains -> [Introduced][ce-598] in GitLab 7.11.0 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/598) in GitLab 7.11.0 You can restrict users to only sign up using email addresses matching the given domains list. ## Blacklist email domains -> [Introduced][ce-5259] in GitLab 8.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5259) in GitLab 8.10. With this feature enabled, you can block email addresses of a specific domain from creating an account on your GitLab server. This is particularly useful @@ -94,6 +94,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -[ce-5259]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5259 -[ce-598]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/598 diff --git a/doc/user/admin_area/settings/terms.md b/doc/user/admin_area/settings/terms.md index d89381fd810..77c172aa927 100644 --- a/doc/user/admin_area/settings/terms.md +++ b/doc/user/admin_area/settings/terms.md @@ -8,6 +8,8 @@ type: reference An admin can enforce acceptance of a terms of service and privacy policy. When this option is enabled, new and existing users must accept the terms. +If configured, the Terms of Service page can be viewed via `https://your-instance.com/-/users/terms` at anytime. + ## Configuration To enforce acceptance of a Terms of Service and Privacy Policy: @@ -21,7 +23,7 @@ To enforce acceptance of a Terms of Service and Privacy Policy: 1. Click **Save changes**. 1. When you are presented with the **Terms of Service** statement, click **Accept terms**. -. + For each update to the terms, a new version is stored. When a user accepts or declines the terms, GitLab will record which version they accepted or declined. diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index 7869f7de1b6..f3eb094887e 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -58,75 +58,7 @@ sequenceDiagram ## Usage Ping **(CORE ONLY)** -> - [Introduced][ee-557] in GitLab Enterprise Edition 8.10. -> - More statistics [were added][ee-735] in GitLab Enterprise Edition 8.12. -> - [Moved to GitLab Core][ce-23361] in 9.1. -> - More statistics [were added][ee-6602] in GitLab Ultimate 11.2. - -GitLab sends a weekly payload containing usage data to GitLab Inc. The usage -ping uses high-level data to help our product, support, and sales teams. It does -not send any project names, usernames, or any other specific data. The -information from the usage ping is not anonymous, it is linked to the hostname -of the instance. - -You can view the exact JSON payload in the administration panel. To view the payload: - -1. Navigate to the **Admin Area > Settings > Metrics and profiling**. -1. Expand the **Usage statistics** section. -1. Click the **Preview payload** button. - -You can see how [the usage ping data maps to different stages of the product](https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/version_usage_stats_to_stage_mappings.csv). - -Usage ping is important to GitLab as we use it to calculate our [Action Monthly Active Users (AMAU)](https://about.gitlab.com/handbook/product/metrics/#action-monthly-active-users-amau) which helps us measure the success of our features. - -### Request flow example - -The following example shows a basic request/response flow between the self-managed GitLab instance, GitLab Version Application, -GitLab License Application and Salesforce: - -```mermaid -sequenceDiagram - participant GitLab instance - participant Version Application - participant License Application - participant Salesforce - GitLab instance->>Version Application: Usage Ping data - loop Process Usage Data - Version Application->>Version Application: Parse Usage Data - Version Application->>Version Application: Record Usage Data - Version Application->>Version Application: Update license ping time - end - Version Application-xLicense Application: Request Zuora subscription id - License Application-xVersion Application: Zuora subscription id - Version Application-xSalesforce: Request Zuora account id by Zuora subscription id - Salesforce-xVersion Application: Zuora account id - Version Application-xSalesforce: Usage data for the Zuora account - Version Application->>GitLab instance: Conversational Development Index -``` - -### Deactivate the usage ping - -The usage ping is opt-out. If you want to deactivate this feature, go to -the Settings page of your administration panel and uncheck the Usage ping -checkbox. - -To disable the usage ping and prevent it from being configured in future through -the administration panel, Omnibus installs can set the following in -[`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): - -```ruby -gitlab_rails['usage_ping_enabled'] = false -``` - -And source installs can set the following in `gitlab.yml`: - -```yaml -production: &base - # ... - gitlab: - # ... - usage_ping_enabled: false -``` +See [Usage Ping guide](../../../development/telemetry/usage_ping.md). ## Instance statistics visibility **(CORE ONLY)** @@ -148,307 +80,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -[ee-557]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/557 -[ee-735]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/735 -[ce-23361]: https://gitlab.com/gitlab-org/gitlab-foss/issues/23361 -[ee-6602]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6602 - -## Usage Statistics Collected - -| Statistic | Section | Stage | Description | -|---|---|---|---| -|uuid||| -|hostname||| -|version||| -|installation_type||| -|active_user_count||| -|recorded_at||| -|edition||| -|license_md5||| -|license_id||| -|historical_max_users||| -|Name|licensee|| -|Email|licensee|| -|Company|licensee|| -|license_user_count||| -|license_starts_at||| -|license_expires_at||| -|license_plan||| -|license_trial||| -|assignee_lists|counts|| -|boards|counts|| -|ci_builds|counts|| -|ci_internal_pipelines|counts|| -|ci_external_pipelines|counts|| -|ci_pipeline_config_auto_devops|counts|| -|ci_pipeline_config_repository|counts|| -|ci_runners|counts|| -|ci_triggers|counts|| -|ci_pipeline_schedules|counts|| -|auto_devops_enabled|counts|| -|auto_devops_disabled|counts|| -|deploy_keys|counts|| -|deployments|counts|| -|successful_deployments|counts|| -|failed_deployments|counts|| -|environments|counts|| -|clusters|counts|| -|clusters_enabled|counts|| -|project_clusters_enabled|counts|| -|group_clusters_enabled|counts|| -|instance_clusters_enabled|counts|| -|clusters_disabled|counts|| -|project_clusters_disabled|counts|| -|group_clusters_disabled|counts|| -|instance_clusters_disabled|counts|| -|clusters_platforms_eks|counts|| -|clusters_platforms_gke|counts|| -|clusters_platforms_user|counts|| -|clusters_applications_helm|counts|| -|clusters_applications_ingress|counts|| -|clusters_applications_cert_managers|counts|| -|clusters_applications_crossplane|counts|| -|clusters_applications_prometheus|counts|| -|clusters_applications_runner|counts|| -|clusters_applications_knative|counts|| -|clusters_applications_elastic_stack|counts|| -|clusters_management_project|counts|| -|in_review_folder|counts|| -|grafana_integrated_projects|counts|| -|groups|counts|| -|issues|counts|| -|issues_created_from_gitlab_error_tracking_ui|counts|| -|issues_with_associated_zoom_link|counts|| -|issues_using_zoom_quick_actions|counts|| -|issues_with_embedded_grafana_charts_approx|counts|| -|issues_with_health_status|counts|| -|keys|counts|| -|label_lists|counts|| -|lfs_objects|counts|| -|milestone_lists|counts|| -|milestones|counts|| -|pages_domains|counts|| -|pool_repositories|counts|| -|projects|counts|| -|projects_imported_from_github|counts|| -|projects_with_repositories_enabled|counts|| -|projects_with_error_tracking_enabled|counts|| -|protected_branches|counts|| -|releases|counts|| -|remote_mirrors|counts|| -|snippets|counts|| -|suggestions|counts|| -|todos|counts|| -|uploads|counts|| -|web_hooks|counts|| -|projects_alerts_active|counts|| -|projects_asana_active|counts|| -|projects_assembla_active|counts|| -|projects_bamboo_active|counts|| -|projects_bugzilla_active|counts|| -|projects_buildkite_active|counts|| -|projects_campfire_active|counts|| -|projects_custom_issue_tracker_active|counts|| -|projects_discord_active|counts|| -|projects_drone_ci_active|counts|| -|projects_emails_on_push_active|counts|| -|projects_external_wiki_active|counts|| -|projects_flowdock_active|counts|| -|projects_github_active|counts|| -|projects_hangouts_chat_active|counts|| -|projects_hipchat_active|counts|| -|projects_irker_active|counts|| -|projects_jenkins_active|counts|| -|projects_jenkins_deprecated_active|counts|| -|projects_jira_active -|counts|| -|projects_mattermost_active|counts|| -|projects_mattermost_slash_commands_active|counts|| -|projects_microsoft_teams_active|counts|| -|projects_packagist_active|counts|| -|projects_pipelines_email_active|counts|| -|projects_pivotaltracker_active|counts|| -|projects_prometheus_active|counts|| -|projects_pushover_active|counts|| -|projects_redmine_active|counts|| -|projects_slack_active|counts|| -|projects_slack_slash_commands_active|counts|| -|projects_teamcity_active|counts|| -|projects_unify_circuit_active|counts|| -|projects_youtrack_active|counts|| -|projects_slack_notifications_active|counts|| -|projects_slack_slash_active|counts|| -|projects_jira_server_active|counts|| -|projects_jira_cloud_active|counts|| -|projects_jira_dvcs_cloud_active|counts|| -|projects_jira_dvcs_server_active|counts|| -|labels|counts|| -|merge_requests|counts|| -|notes|counts|| -|wiki_pages_create|counts|| -|wiki_pages_update|counts|| -|wiki_pages_delete|counts|| -|web_ide_commits|counts|| -|web_ide_views|counts|| -|web_ide_merge_requests|counts|| -|web_ide_previews|counts|| -|snippet_comment|counts|| -|commit_comment|counts|| -|merge_request_comment|counts|| -|snippet_create|counts|| -|snippet_update|counts|| -|navbar_searches|counts|| -|cycle_analytics_views|counts|| -|productivity_analytics_views|counts|| -|source_code_pushes|counts|| -|merge_request_create|counts|| -|design_management_designs_create|counts|| -|design_management_designs_update|counts|| -|design_management_designs_delete|counts|| -|licenses_list_views|counts|| -|user_preferences_group_overview_details|counts|| -|user_preferences_group_overview_security_dashboard|counts|| -|ingress_modsecurity_blocking|counts|| -|ingress_modsecurity_disabled|counts|| -|dependency_list_usages_total|counts|| -|epics|counts|| -|feature_flags|counts|| -|geo_nodes|counts|| -|incident_issues|counts|| -|ldap_group_links|counts|| -|ldap_keys|counts|| -|ldap_users|counts|| -|pod_logs_usages_total|counts|| -|projects_enforcing_code_owner_approval|counts|| -|projects_mirrored_with_pipelines_enabled|counts|| -|projects_reporting_ci_cd_back_to_github|counts|| -|projects_with_packages|counts|| -|projects_with_prometheus_alerts|counts|| -|projects_with_tracing_enabled|counts|| -|projects_with_alerts_service_enabled|counts|| -|template_repositories|counts|| -|container_scanning_jobs|counts|| -|dependency_scanning_jobs|counts|| -|license_management_jobs|counts|| -|sast_jobs|counts|| -|status_page_projects|counts|monitor| -|status_page_issues|counts|monitor| -|epics_deepest_relationship_level|counts|| -|operations_dashboard_default_dashboard|counts|| -|operations_dashboard_users_with_projects_added|counts|| -|container_registry_enabled||| -|dependency_proxy_enabled||| -|gitlab_shared_runners_enabled||| -|gravatar_enabled||| -|influxdb_metrics_enabled||| -|ldap_enabled||| -|mattermost_enabled||| -|omniauth_enabled||| -|prometheus_metrics_enabled||| -|reply_by_email_enabled||| -|signup_enabled||| -|web_ide_clientside_preview_enabled||| -|ingress_modsecurity_enabled||| -|elasticsearch_enabled||| -|license_trial_ends_on||| -|geo_enabled||| -|version|Git|| -|version|Gitaly|| -|servers|Gitaly|| -|filesystems|Gitaly|| -|enabled|gitlab_pages|| -|version|gitlab_pages|| -|adapter|database|| -|version|database|| -|average|avg_cycle_analytics - issue|| -|sd|avg_cycle_analytics - issue|| -|missing|avg_cycle_analytics - issue|| -|average|avg_cycle_analytics - plan|| -|sd|avg_cycle_analytics - plan|| -|missing|avg_cycle_analytics - plan|| -|average|avg_cycle_analytics - code|| -|sd|avg_cycle_analytics - code|| -|missing|avg_cycle_analytics - code|| -|average|avg_cycle_analytics - test|| -|sd|avg_cycle_analytics - test|| -|missing|avg_cycle_analytics - test|| -|average|avg_cycle_analytics - review|| -|sd|avg_cycle_analytics - review|| -|missing|avg_cycle_analytics - review|| -|average|avg_cycle_analytics - staging|| -|sd|avg_cycle_analytics - staging|| -|missing|avg_cycle_analytics - staging|| -|average|avg_cycle_analytics - production|| -|sd|avg_cycle_analytics - production|| -|missing|avg_cycle_analytics - production|| -|total|avg_cycle_analytics|| -|clusters_applications_cert_managers|usage_activity_by_stage|configure| -|clusters_applications_helm|usage_activity_by_stage|configure| -|clusters_applications_ingress|usage_activity_by_stage|configure| -|clusters_applications_knative|usage_activity_by_stage|configure| -|clusters_management_project|usage_activity_by_stage|configure| -|clusters_disabled|usage_activity_by_stage|configure| -|clusters_enabled|usage_activity_by_stage|configure| -|clusters_platforms_gke|usage_activity_by_stage|configure| -|clusters_platforms_eks|usage_activity_by_stage|configure| -|clusters_platforms_user|usage_activity_by_stage|configure| -|instance_clusters_disabled|usage_activity_by_stage|configure| -|instance_clusters_enabled|usage_activity_by_stage|configure| -|group_clusters_disabled|usage_activity_by_stage|configure| -|group_clusters_enabled|usage_activity_by_stage|configure| -|project_clusters_disabled|usage_activity_by_stage|configure| -|project_clusters_enabled|usage_activity_by_stage|configure| -|projects_slack_notifications_active|usage_activity_by_stage|configure| -|projects_slack_slash_active|usage_activity_by_stage|configure| -|projects_with_prometheus_alerts: 0|usage_activity_by_stage|configure| -|deploy_keys|usage_activity_by_stage|create| -|keys|usage_activity_by_stage|create| -|merge_requests|usage_activity_by_stage|create| -|projects_enforcing_code_owner_approval|usage_activity_by_stage|create| -|projects_imported_from_github|usage_activity_by_stage|create| -|projects_with_repositories_enabled|usage_activity_by_stage|create| -|protected_branches|usage_activity_by_stage|create| -|remote_mirrors|usage_activity_by_stage|create| -|snippets|usage_activity_by_stage|create| -|suggestions:|usage_activity_by_stage|create| -|groups|usage_activity_by_stage|manage| -|ldap_keys|usage_activity_by_stage|manage| -|ldap_users: 0|usage_activity_by_stage|manage| -|users_created|usage_activity_by_stage|manage| -|clusters|usage_activity_by_stage|monitor| -|clusters_applications_prometheus|usage_activity_by_stage|monitor| -|operations_dashboard_default_dashboard|usage_activity_by_stage|monitor| -|operations_dashboard_users_with_projects_added|usage_activity_by_stage|monitor| -|projects_prometheus_active|usage_activity_by_stage|monitor| -|projects_with_error_tracking_enabled|usage_activity_by_stage|monitor| -|projects_with_tracing_enabled: 0|usage_activity_by_stage|monitor| -|projects_with_packages: 0|usage_activity_by_stage|package| -|assignee_lists|usage_activity_by_stage|plan| -|epics|usage_activity_by_stage|plan| -|issues|usage_activity_by_stage|plan| -|label_lists|usage_activity_by_stage|plan| -|milestone_lists|usage_activity_by_stage|plan| -|notes|usage_activity_by_stage|plan| -|projects|usage_activity_by_stage|plan| -|projects_jira_active|usage_activity_by_stage|plan| -|projects_jira_dvcs_cloud_active|usage_activity_by_stage|plan| -|projects_jira_dvcs_server_active|usage_activity_by_stage|plan| -|service_desk_enabled_projects|usage_activity_by_stage|plan| -|service_desk_issues|usage_activity_by_stage|plan| -|todos: 0|usage_activity_by_stage|plan| -|deployments|usage_activity_by_stage|release| -|failed_deployments|usage_activity_by_stage|release| -|projects_mirrored_with_pipelines_enabled|usage_activity_by_stage|release| -|releases|usage_activity_by_stage|release| -|successful_deployments: 0|usage_activity_by_stage|release| -|user_preferences_group_overview_security_dashboard: 0|usage_activity_by_stage|secure| -|ci_builds|usage_activity_by_stage|verify| -|ci_external_pipelines|usage_activity_by_stage|verify| -|ci_internal_pipelines|usage_activity_by_stage|verify| -|ci_pipeline_config_auto_devops|usage_activity_by_stage|verify| -|ci_pipeline_config_repository|usage_activity_by_stage|verify| -|ci_pipeline_schedules|usage_activity_by_stage|verify| -|ci_pipelines|usage_activity_by_stage|verify| -|ci_triggers|usage_activity_by_stage|verify| -|clusters_applications_runner|usage_activity_by_stage|verify| -|projects_reporting_ci_cd_back_to_github: 0|usage_activity_by_stage|verify| diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index f827fed833b..95748f68a55 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -28,6 +28,21 @@ For more details, see [Protected branches](../../project/protected_branches.md). To change this setting for a specific group, see [Default branch protection for groups](../../group/index.md#changing-the-default-branch-protection-of-a-group) +### Disable group owners from updating default branch protection **(PREMIUM ONLY)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211944) in GitLab 13.0. + +By default, group owners are allowed to override the branch protection set at the global level. + +In [GitLab Premium or higher](https://about.gitlab.com/pricing/), GitLab administrators can disable this privilege of group owners. + +To do this: + +1. Uncheck the **Allow owners to manage default branch protection per group** checkbox. + +NOTE: **Note:** +GitLab administrators can still update the default branch protection of a group. + ## Default project creation protection Project creation protection specifies which roles can create projects. @@ -62,6 +77,13 @@ To change this period: 1. Select the desired option. 1. Click **Save changes**. +### Override default deletion adjourned period + +Alternatively, projects that are marked for removal can be deleted immediately. To do so: + +1. [Restore the project](../../project/settings/#restore-a-project-premium). +1. Delete the project as described in the [Administering Projects page](../../admin_area/#administering-projects). + ## Default project visibility To set the default visibility levels for new projects: diff --git a/doc/user/analytics/code_review_analytics.md b/doc/user/analytics/code_review_analytics.md index bb74e673b56..e0aa01c29b2 100644 --- a/doc/user/analytics/code_review_analytics.md +++ b/doc/user/analytics/code_review_analytics.md @@ -1,7 +1,12 @@ --- description: "Learn how long your open merge requests have spent in code review, and what distinguishes the longest-running." # Up to ~200 chars long. They will be displayed in Google Search snippets. It may help to write the page intro first, and then reuse it here. +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- + # Code Review Analytics **(STARTER)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/38062) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.7. diff --git a/doc/user/analytics/img/repository_analytics_v13_0.png b/doc/user/analytics/img/repository_analytics_v13_0.png Binary files differnew file mode 100644 index 00000000000..b70b63a6239 --- /dev/null +++ b/doc/user/analytics/img/repository_analytics_v13_0.png diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index b2f7da234ad..48df7bc340a 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -1,3 +1,10 @@ +--- +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Analytics ## Analytics workspace @@ -32,6 +39,6 @@ The following analytics features are available at the project level: - [Code Review](code_review_analytics.md). **(STARTER)** - [Insights](../group/insights/index.md). **(ULTIMATE)** - [Issues](../group/issues_analytics/index.md). **(PREMIUM)** -- Repository. **(STARTER)** +- [Repository](repository_analytics.md). - [Value Stream](value_stream_analytics.md), enabled with the `cycle_analytics` [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). **(STARTER)** diff --git a/doc/user/analytics/productivity_analytics.md b/doc/user/analytics/productivity_analytics.md index 0fa990150d7..d0fda61d6a5 100644 --- a/doc/user/analytics/productivity_analytics.md +++ b/doc/user/analytics/productivity_analytics.md @@ -1,3 +1,10 @@ +--- +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Productivity Analytics **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12079) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. diff --git a/doc/user/analytics/repository_analytics.md b/doc/user/analytics/repository_analytics.md new file mode 100644 index 00000000000..17032990b09 --- /dev/null +++ b/doc/user/analytics/repository_analytics.md @@ -0,0 +1,40 @@ +--- +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Repository Analytics + +Get high-level overview of the project's Git repository. + + + +## Availability + +Repository Analytics is part of [GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-foss). It's available to anyone who has permission to clone the repository. + +The feature requires: + +- An initialized Git repository. +- At least one commit in the default branch (`master` by default). + +## Overview + +You can find Repository Analytics in the project's sidebar. To access the page, go to **{chart}** **Analytics > Repository**. + +NOTE: **Note:** +Without a Git commit in the default branch, the menu item won't be visible. + +### Charts + +The data in the charts are updated soon after each commit in the default branch. + +Available charts: + +- Programming languages used in the repository +- Commit statistics (last month) +- Commits per day of month +- Commits per weekday +- Commits per day hour (UTC) diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index 32f393d342b..90a4f96f00b 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -1,3 +1,10 @@ +--- +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Value Stream Analytics > - Introduced as Cycle Analytics prior to GitLab 12.3 at the project level. @@ -11,9 +18,6 @@ spent in each stage defined in the process. For information on how to contribute to the development of Value Stream Analytics, see our [contributor documentation](../../development/value_stream_analytics.md). -NOTE: **Note:** -Use the `cycle_analytics` feature flag to enable at the group level. - Value Stream Analytics is useful in order to quickly determine the velocity of a given project. It points to bottlenecks in the development process, enabling management to uncover, triage, and identify the root cause of slowdowns in the software development life cycle. @@ -26,7 +30,7 @@ calculates a separate median for each stage. Value Stream Analytics is available: - From GitLab 12.9, at the group level via **Group > Analytics > Value Stream**. **(PREMIUM)** -- At the project level via **Project > Value Stream Analytics**. +- At the project level via **Project > Analytics > Value Stream**. There are seven stages that are tracked as part of the Value Stream Analytics calculations. @@ -69,7 +73,7 @@ Each stage of Value Stream Analytics is further described in the table below. | **Stage** | **Description** | | --------- | --------------- | -| Issue | Measures the median time between creating an issue and taking action to solve it, by either labeling it or adding it to a milestone, whatever comes first. The label will be tracked only if it already has an [Issue Board list](../project/issue_board.md#creating-a-new-list) created for it. | +| Issue | Measures the median time between creating an issue and taking action to solve it, by either labeling it or adding it to a milestone, whatever comes first. The label will be tracked only if it already has an [Issue Board list](../project/issue_board.md) created for it. | | Plan | Measures the median time between the action you took for the previous stage, and pushing the first commit to the branch. The very first commit of the branch is the one that triggers the separation between **Plan** and **Code**, and at least one of the commits in the branch needs to contain the related issue number (e.g., `#42`). If none of the commits in the branch mention the related issue number, it is not considered to the measurement time of the stage. | | Code | Measures the median time between pushing a first commit (previous stage) and creating a merge request (MR) related to that commit. The key to keep the process tracked is to include the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) to the description of the merge request (for example, `Closes #xxx`, where `xxx` is the number of the issue related to this merge request). If the issue closing pattern is not present in the merge request description, the MR is not considered to the measurement time of the stage. | | Test | Measures the median time to run the entire pipeline for that project. It's related to the time GitLab CI/CD takes to run every job for the commits pushed to that merge request defined in the previous stage. It is basically the start->finish time for all pipelines. | @@ -165,6 +169,21 @@ development workflow. NOTE: **Note:** Customizability is [only available for group-level](https://gitlab.com/gitlab-org/gitlab/-/issues/35823#note_272558950) Value Stream Analytics. +### Stage path + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210315) in GitLab 13.0. + +Stages are visually depicted as a horizontal process flow. Selecting a stage will update the +the content below the value stream. + +This is disabled by default. If you have a self-managed instance, an +administrator can [open a Rails console](../../administration/troubleshooting/navigating_gitlab_via_rails_console.md) +and enable it with the following command: + +```ruby +Feature.enable(:value_stream_analytics_path_navigation) +``` + ### Adding a stage In the following example we're creating a new stage that measures and tracks issues from creation @@ -293,15 +312,6 @@ toggled to show data for merge requests and further refined for specific group-l By default the top group-level labels (max. 10) are pre-selected, with the ability to select up to a total of 15 labels. -### Disabling chart - -This chart is enabled by default. If you have a self-managed instance, an -administrator can open a Rails console and disable it with the following command: - -```ruby -Feature.disable(:tasks_by_type_chart) -``` - ## Permissions The current permissions on the Project Value Stream Analytics dashboard are: @@ -324,14 +334,6 @@ For Value Stream Analytics functionality introduced in GitLab 12.3 and later: - Features are available only on [Premium or Silver tiers](https://about.gitlab.com/pricing/) and above. -## Troubleshooting - -If you see an error as listed in the following table, try the noted solution: - -| Error | Solution | -|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| There was an error fetching the top labels. | Manually enable tasks by type feature in the [rails console](../../administration/troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session), specifically `Feature.enable(:tasks_by_type_chart)`. | - ## More resources Learn more about Value Stream Analytics in the following resources: diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index 131247910ab..dfddbde379d 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -11,7 +11,7 @@ type: reference, howto The security configuration page displays the configuration state of each of the security features and can be accessed through a project's sidebar nav. - + The page uses the project's latest default branch [CI pipeline](../../../ci/pipelines/index.md) to determine the configuration state of each feature. If a job with the expected security report artifact exists in the pipeline, diff --git a/doc/user/application_security/container_scanning/img/container_scanning_v12_9.png b/doc/user/application_security/container_scanning/img/container_scanning_v12_9.png Binary files differdeleted file mode 100644 index 13cacc6a489..00000000000 --- a/doc/user/application_security/container_scanning/img/container_scanning_v12_9.png +++ /dev/null diff --git a/doc/user/application_security/container_scanning/img/container_scanning_v13_0.png b/doc/user/application_security/container_scanning/img/container_scanning_v13_0.png Binary files differnew file mode 100644 index 00000000000..7a079a65072 --- /dev/null +++ b/doc/user/application_security/container_scanning/img/container_scanning_v13_0.png diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 68ad2d427dd..6e52d7dbdcf 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -22,7 +22,7 @@ GitLab checks the Container Scanning report, compares the found vulnerabilities between the source and target branches, and shows the information right on the merge request. - + ## Contribute your scanner @@ -59,7 +59,7 @@ To enable Container Scanning in your pipeline, you need: [predefined environment variables](../../../ci/variables/predefined_variables.md) as defined below: - ```text + ```plaintext $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA ``` @@ -103,7 +103,7 @@ The included template will: and scan it for possible vulnerabilities. The results will be saved as a -[Container Scanning report artifact](../../../ci/yaml/README.md#artifactsreportscontainer_scanning-ultimate) +[Container Scanning report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscontainer_scanning-ultimate) that you can later download and analyze. Due to implementation limitations, we always take the latest Container Scanning artifact available. Behind the scenes, the @@ -169,6 +169,7 @@ using environment variables. | Environment Variable | Description | Default | | ------ | ------ | ------ | +| `SECURE_ANALYZERS_PREFIX` | Set the Docker registry base address from which to download the analyzer. | `"registry.gitlab.com/gitlab-org/security-products/analyzers"` | | `KLAR_TRACE` | Set to true to enable more verbose output from klar. | `"false"` | | `CLAIR_TRACE` | Set to true to enable more verbose output from the clair server process. | `"false"` | | `DOCKER_USER` | Username for accessing a Docker registry requiring authentication. | `$CI_REGISTRY_USER` | @@ -179,11 +180,11 @@ using environment variables. | `CLAIR_VULNERABILITIES_DB_URL` | (**DEPRECATED - use `CLAIR_DB_CONNECTION_STRING` instead**) This variable is explicitly set in the [services section](https://gitlab.com/gitlab-org/gitlab/-/blob/898c5da43504eba87b749625da50098d345b60d6/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L23) of the `Container-Scanning.gitlab-ci.yml` file and defaults to `clair-vulnerabilities-db`. This value represents the address that the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db) is running on and **shouldn't be changed** unless you're running the image locally as described in the [Running the standalone Container Scanning Tool](#running-the-standalone-container-scanning-tool) section. | `clair-vulnerabilities-db` | | `CLAIR_DB_CONNECTION_STRING` | This variable represents the [connection string](https://www.postgresql.org/docs/9.3/libpq-connect.html#AEN39692) to the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db) database and **shouldn't be changed** unless you're running the image locally as described in the [Running the standalone Container Scanning Tool](#running-the-standalone-container-scanning-tool) section. The host value for the connection string must match the [alias](https://gitlab.com/gitlab-org/gitlab/-/blob/898c5da43504eba87b749625da50098d345b60d6/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L23) value of the `Container-Scanning.gitlab-ci.yml` template file, which defaults to `clair-vulnerabilities-db`. | `postgresql://postgres:password@clair-vulnerabilities-db:5432/postgres?sslmode=disable&statement_timeout=60000` | | `CI_APPLICATION_REPOSITORY` | Docker repository URL for the image to be scanned. | `$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG` | -| `CI_APPLICATION_TAG` | Docker respository tag for the image to be scanned. | `$CI_COMMIT_SHA` | +| `CI_APPLICATION_TAG` | Docker repository tag for the image to be scanned. | `$CI_COMMIT_SHA` | | `CLAIR_DB_IMAGE` | The Docker image name and tag for the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes, or to refer to a locally hosted vulnerabilities database for an on-premise offline installation. | `arminc/clair-db:latest` | | `CLAIR_DB_IMAGE_TAG` | (**DEPRECATED - use `CLAIR_DB_IMAGE` instead**) The Docker image tag for the [PostgreSQL server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes. | `latest` | | `DOCKERFILE_PATH` | The path to the `Dockerfile` to be used for generating remediations. By default, the scanner will look for a file named `Dockerfile` in the root directory of the project, so this variable should only be configured if your `Dockerfile` is in a non-standard location, such as a subdirectory. See [Solutions for vulnerabilities](#solutions-for-vulnerabilities-auto-remediation) for more details. | `Dockerfile` | -| `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs that you want to trust. | "" | +| `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs that you want to trust. | "" | ### Overriding the Container Scanning template @@ -229,25 +230,29 @@ To use Container Scanning in an offline environment, you need: NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), -meaning the runner may try to pull remote images even if a local copy is available. Set GitLab -Runner's [`pull_policy` to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) -in an offline environment if you prefer using only locally available Docker images. +meaning the Runner tries to pull Docker images from the GitLab container registry even if a local +copy is available. GitLab Runner's [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) +in an offline environment if you prefer using only locally available Docker images. However, we +recommend keeping the pull policy setting to `always` if not in an offline environment, as this +enables the use of updated scanners in your CI/CD pipelines. #### Make GitLab Container Scanning analyzer images available inside your Docker registry -For Container Scanning, import and host the following images from `registry.gitlab.com` to your -offline [local Docker container registry](../../packages/container_registry/index.md): +For Container Scanning, import the following default images from `registry.gitlab.com` into your +[local Docker container registry](../../packages/container_registry/index.md): -- [arminc/clair-db vulnerabilities database](https://hub.docker.com/r/arminc/clair-db) -- GitLab klar analyzer: `registry.gitlab.com/gitlab-org/security-products/analyzers/klar` +```plaintext +registry.gitlab.com/gitlab-org/security-products/analyzers/klar +https://hub.docker.com/r/arminc/clair-db +``` The process for importing Docker images into a local offline Docker registry depends on **your network security policy**. Please consult your IT staff to find an accepted and approved -process by which external resources can be imported or temporarily accessed. - -Note that these scanners are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) +process by which you can import or temporarily access external resources. Note that these scanners +are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) with new definitions, so consider if you are able to make periodic updates yourself. -You can read more specific steps on how to do this [below](#automating-container-scanning-vulnerability-database-updates-with-a-pipeline). + +For more information, see [the specific steps on how to update an image with a pipeline](#automating-container-scanning-vulnerability-database-updates-with-a-pipeline). For details on saving and transporting Docker images as a file, see Docker's documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), @@ -255,8 +260,6 @@ For details on saving and transporting Docker images as a file, see Docker's doc #### Set Container Scanning CI job variables to use local Container Scanner analyzers -Container Scanning can be executed on an offline GitLab Ultimate installation using the following process: - 1. [Override the container scanning template](#overriding-the-container-scanning-template) in your `.gitlab-ci.yml` file to refer to the Docker images hosted on your local Docker container registry: ```yaml @@ -412,11 +415,11 @@ the report JSON unless stated otherwise. Presence of optional fields depends on | `vulnerabilities[].message` | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. | | `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | | `vulnerabilities[].cve` | (**DEPRECATED - use `vulnerabilities[].id` instead**) A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | -| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) only provides the following levels: `Unknown`, `Low`, `Medium`, `High`, `Critical`. | -| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) does not provide a confidence level, so this value is currently hardcoded to `Unknown`. | +| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) only provides the following levels: `Unknown`, `Low`, `Medium`, `High`, `Critical`. | +| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) does not provide a confidence level, so this value is currently hardcoded to `Unknown`. | | `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | | `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | -| `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. | +| `vulnerabilities[].scanner.id` | ID of the scanner as a snake_case string. | | `vulnerabilities[].scanner.name` | Name of the scanner, for display purposes. | | `vulnerabilities[].location` | A node that tells where the vulnerability is located. | | `vulnerabilities[].location.dependency` | A node that describes the dependency of a project where the vulnerability is located. | @@ -435,7 +438,7 @@ the report JSON unless stated otherwise. Presence of optional fields depends on | `vulnerabilities[].links[].url` | URL of the vulnerability details document. Optional. | | `remediations` | An array of objects containing information on cured vulnerabilities along with patch diffs to apply. Empty if no remediations provided by an underlying analyzer. | | `remediations[].fixes` | An array of strings that represent references to vulnerabilities fixed by this particular remediation. | -| `remediations[].fixes[].id` | The id of a fixed vulnerability. | +| `remediations[].fixes[].id` | The ID of a fixed vulnerability. | | `remediations[].fixes[].cve` | (**DEPRECATED - use `remediations[].fixes[].id` instead**) A string value that describes a fixed vulnerability in the same format as `vulnerabilities[].cve`. | | `remediations[].summary` | Overview of how the vulnerabilities have been fixed. | | `remediations[].diff` | base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). | @@ -476,7 +479,7 @@ When the GitLab Runner uses the Docker executor and NFS is used (for example, `/var/lib/docker` is on an NFS mount), Container Scanning might fail with an error like the following: -```text +```plaintext docker: Error response from daemon: failed to copy xattrs: failed to set xattr "security.selinux" on /path/to/file: operation not supported. ``` diff --git a/doc/user/application_security/dast/img/dast_all_v12_9.png b/doc/user/application_security/dast/img/dast_all_v12_9.png Binary files differdeleted file mode 100644 index 548cea3f7f9..00000000000 --- a/doc/user/application_security/dast/img/dast_all_v12_9.png +++ /dev/null diff --git a/doc/user/application_security/dast/img/dast_all_v13_0.png b/doc/user/application_security/dast/img/dast_all_v13_0.png Binary files differnew file mode 100644 index 00000000000..7b67fc44fae --- /dev/null +++ b/doc/user/application_security/dast/img/dast_all_v13_0.png diff --git a/doc/user/application_security/dast/img/dast_single_v12_9.png b/doc/user/application_security/dast/img/dast_single_v12_9.png Binary files differdeleted file mode 100644 index a8a4b1c1d4f..00000000000 --- a/doc/user/application_security/dast/img/dast_single_v12_9.png +++ /dev/null diff --git a/doc/user/application_security/dast/img/dast_single_v13_0.png b/doc/user/application_security/dast/img/dast_single_v13_0.png Binary files differnew file mode 100644 index 00000000000..91ed4d916ab --- /dev/null +++ b/doc/user/application_security/dast/img/dast_single_v13_0.png diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index abf194aae48..1d5f96d96bb 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -19,42 +19,48 @@ Dynamic Application Security Testing (DAST) comes into place. ## Overview -If you are using [GitLab CI/CD](../../../ci/README.md), you can analyze your running web application(s) +If you're using [GitLab CI/CD](../../../ci/README.md), you can analyze your running web applications for known vulnerabilities using Dynamic Application Security Testing (DAST). - You can take advantage of DAST by either [including the CI job](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using -[Auto DAST](../../../topics/autodevops/stages.md#auto-dast-ultimate) -that is provided by [Auto DevOps](../../../topics/autodevops/index.md). +[Auto DAST](../../../topics/autodevops/stages.md#auto-dast-ultimate), +provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the DAST report, compares the found vulnerabilities between the source and target -branches, and shows the information right on the merge request. +branches, and shows the information on the merge request. NOTE: **Note:** This comparison logic uses only the latest pipeline executed for the target branch's base commit. Running the pipeline on any other commit has no effect on the merge request. - + -By clicking on one of the detected linked vulnerabilities, you will be able to +By clicking on one of the detected linked vulnerabilities, you can see the details and the URL(s) affected. - + [Dynamic Application Security Testing (DAST)](https://en.wikipedia.org/wiki/Dynamic_Application_Security_Testing) -is using the popular open source tool [OWASP ZAProxy](https://github.com/zaproxy/zaproxy) +uses the popular open source tool [OWASP ZAProxy](https://github.com/zaproxy/zaproxy) to perform an analysis on your running web application. -By default, DAST executes [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan) and will perform passive scanning only. It will not actively attack your application. - +By default, DAST executes [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan) +and performs passive scanning only. It won't actively attack your application. However, DAST can be [configured](#full-scan) -to also perform a so-called "active scan". That is, attack your application and produce a more extensive security report. +to also perform an *active scan*: attack your application and produce a more extensive security report. It can be very useful combined with [Review Apps](../../../ci/review_apps/index.md). +NOTE: **Note:** +A pipeline may consist of multiple jobs, including SAST and DAST scanning. If any +job fails to finish for any reason, the security dashboard won't show DAST scanner +output. For example, if the DAST job finishes but the SAST job fails, the security +dashboard won't show DAST results. The analyzer will output an +[exit code](../../../development/integrations/secure.md#exit-code) on failure. + ## Use cases It helps you automatically find security vulnerabilities in your running web -applications while you are developing and testing your applications. +applications while you're developing and testing your applications. ## Requirements @@ -66,9 +72,8 @@ To run a DAST job, you need GitLab Runner with the For GitLab 11.9 and later, to enable DAST, you must [include](../../../ci/yaml/README.md#includetemplate) the [`DAST.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml) -that's provided as a part of your GitLab installation. -For GitLab versions earlier than 11.9, you can copy and use the job as defined -in that template. +that's provided as a part of your GitLab installation. For GitLab versions earlier +than 11.9, you can copy and use the job as defined in that template. Add the following to your `.gitlab-ci.yml` file: @@ -85,32 +90,39 @@ There are two ways to define the URL to be scanned by DAST: 1. Set the `DAST_WEBSITE` [variable](../../../ci/yaml/README.md#variables). 1. Add it in an `environment_url.txt` file at the root of your project. - This is great for testing in dynamic environments. In order to run DAST against - an app that is dynamically created during a GitLab CI/CD pipeline, have the app - persist its domain in an `environment_url.txt` file, and DAST will - automatically parse that file to find its scan target. - You can see an [example](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) - of this in our Auto DevOps CI YML. + This is great for testing in dynamic environments. In order to run DAST against + an app dynamically created during a GitLab CI/CD pipeline, have the app + persist its domain in an `environment_url.txt` file, and DAST + automatically parses that file to find its scan target. + You can see an [example](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) + of this in our Auto DevOps CI YAML. -If both values are set, the `DAST_WEBSITE` value will take precedence. +If both values are set, the `DAST_WEBSITE` value takes precedence. -The included template will create a `dast` job in your CI/CD pipeline and scan +The included template creates a `dast` job in your CI/CD pipeline and scans your project's source code for possible vulnerabilities. -The results will be saved as a -[DAST report artifact](../../../ci/yaml/README.md#artifactsreportsdast-ultimate) +The results are saved as a +[DAST report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsdast-ultimate) that you can later download and analyze. Due to implementation limitations we always take the latest DAST artifact available. Behind the scenes, the [GitLab DAST Docker image](https://gitlab.com/gitlab-org/security-products/dast) is used to run the tests on the specified URL and scan it for possible vulnerabilities. -By default, the DAST template will use the latest major version of the DAST Docker image. Using the `DAST_VERSION` variable, -you can choose to automatically update DAST with new features and fixes by pinning to a major version (e.g. 1), only update fixes by pinning to a minor version (e.g. 1.6) or prevent all updates by pinning to a specific version (e.g. 1.6.4). +By default, the DAST template will use the latest major version of the DAST Docker +image. Using the `DAST_VERSION` variable, you can choose how DAST updates: + +- Automatically update DAST with new features and fixes by pinning to a major version (such as `1`). +- Only update fixes by pinning to a minor version (such as `1.6`). +- Prevent all updates by pinning to a specific version (such as `1.6.4`). + Find the latest DAST versions on the [Releases](https://gitlab.com/gitlab-org/security-products/dast/-/releases) page. ### When DAST scans run -When using `DAST.gitlab-ci.yml` template, the `dast` job is run last as shown in the example below. To ensure DAST is scanning the latest code, your CI pipeline should deploy changes to the web server in one of the jobs preceeding the `dast` job. +When using `DAST.gitlab-ci.yml` template, the `dast` job is run last as shown in +the example below. To ensure DAST is scanning the latest code, your CI pipeline +should deploy changes to the web server in one of the jobs preceding the `dast` job. ```yaml stages: @@ -120,12 +132,15 @@ stages: - dast ``` -Be aware that if your pipeline is configured to deploy to the same webserver in each run, running a pipeline while another is still running, could cause a race condition -where one pipeline overwrites the code from another pipeline. The site to be scanned should be excluded from changes for the duration of a DAST scan. -The only changes to the site should be from the DAST scanner. Be aware that any changes that users, scheduled tasks, database or code changes, other pipelines, or other scanners make to +Be aware that if your pipeline is configured to deploy to the same webserver in +each run, running a pipeline while another is still running could cause a race condition +where one pipeline overwrites the code from another pipeline. The site to be scanned +should be excluded from changes for the duration of a DAST scan. +The only changes to the site should be from the DAST scanner. Be aware that any +changes that users, scheduled tasks, database changes, code changes, other pipelines, or other scanners make to the site during a scan could lead to inaccurate results. -### Authenticated scan +### Authentication It's also possible to authenticate the user before performing the DAST checks: @@ -144,12 +159,15 @@ variables: ``` The results will be saved as a -[DAST report artifact](../../../ci/yaml/README.md#artifactsreportsdast-ultimate) +[DAST report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsdast-ultimate) that you can later download and analyze. Due to implementation limitations, we always take the latest DAST artifact available. DANGER: **Danger:** -**DO NOT** run an authenticated scan against a production server. When an authenticated scan is run, it may perform *any* function that the authenticated user can. This includes modifying and deleting data, submitting forms, following links, and so on. Only run an authenticated scan against a test server. +**NEVER** run an authenticated scan against a production server. When an authenticated +scan is run, it may perform *any* function that the authenticated user can. This +includes actions like modifying and deleting data, submitting forms, and following links. +Only run an authenticated scan against a test server. ### Full scan @@ -170,7 +188,8 @@ The DAST job can be run anywhere, which means you can accidentally hit live web and potentially damage them. You could even take down your production environment. For that reason, you should use domain validation. -Domain validation is not required by default. It can be required by setting the [environment variable](#available-variables) `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` to true. +Domain validation is not required by default. It can be required by setting the +[environment variable](#available-variables) `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` to `"true"`. ```yaml include: @@ -181,19 +200,23 @@ variables: DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED: "true" ``` -Since ZAP full scan actively attacks the target application, DAST sends a ping to the target (normally defined in `DAST_WEBSITE` or `environment_url.txt`) beforehand. +Since ZAP full scan actively attacks the target application, DAST sends a ping +to the target (normally defined in `DAST_WEBSITE` or `environment_url.txt`) beforehand. -If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is false or unset, the scan will _proceed_ unless the response to the ping -includes a `Gitlab-DAST-Permission` header with a value of `deny`. +- If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is `false` or unset, the scan will + proceed unless the response to the ping includes a `Gitlab-DAST-Permission` + header with a value of `deny`. +- If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is `true`, the scan will exit + unless the response to the ping includes a `Gitlab-DAST-Permission` header with + a value of `allow`. -If `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` is true, the scan will _exit_ unless the response to the ping -includes a `Gitlab-DAST-Permission` header with a value of `allow`. - -Here are some examples of adding the `Gitlab-DAST-Permission` header to a response in Rails, Django, and Node (with Express). +Here are some examples of adding the `Gitlab-DAST-Permission` header to a response +in Rails, Django, and Node (with Express). ##### Ruby on Rails -Here's how you would add a [custom header in Ruby on Rails](https://guides.rubyonrails.org/action_controller_overview.html#setting-custom-headers): +Here's how you would add a +[custom header in Ruby on Rails](https://guides.rubyonrails.org/action_controller_overview.html#setting-custom-headers): ```ruby class DastWebsiteTargetController < ActionController::Base @@ -207,7 +230,8 @@ end ##### Django -Here's how you would add a [custom header in Django](https://docs.djangoproject.com/en/2.2/ref/request-response/#setting-header-fields): +Here's how you would add a +[custom header in Django](https://docs.djangoproject.com/en/2.2/ref/request-response/#setting-header-fields): ```python class DastWebsiteTargetView(View): @@ -220,7 +244,8 @@ class DastWebsiteTargetView(View): ##### Node (with Express) -Here's how you would add a [custom header in Node (with Express)](http://expressjs.com/en/5x/api.html#res.append): +Here's how you would add a +[custom header in Node (with Express)](http://expressjs.com/en/5x/api.html#res.append): ```javascript app.get('/dast-website-target', function(req, res) { @@ -235,7 +260,8 @@ It's also possible to add the `Gitlab-DAST-Permission` header via a proxy. ###### NGINX -The following config allows NGINX to act as a reverse proxy and add the `Gitlab-DAST-Permission` [header](http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header): +The following configuration allows NGINX to act as a reverse proxy and add the +`Gitlab-DAST-Permission` [header](http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header): ```nginx # default.conf @@ -287,9 +313,9 @@ API scans support OpenAPI V2 and OpenAPI V3 specifications. You can define these #### Import API specification from a URL If your API specification is accessible at a URL, you can pass that URL in directly as the target. -The specification doesn't have to be hosted on the same host as the API being tested. +The specification does not have to be hosted on the same host as the API being tested. -```yml +```yaml include: - template: DAST.gitlab-ci.yml @@ -299,9 +325,11 @@ variables: #### Import API specification from a file -If your API specification is in your repository, you can provide the specification's filename directly as the target. The specification file is expected to be in the `/zap/wrk` directory. +If your API specification is in your repository, you can provide the specification's +filename directly as the target. The specification file is expected to be in the +`/zap/wrk` directory. -```yml +```yaml dast: script: - mkdir -p /zap/wrk @@ -314,23 +342,27 @@ dast: #### Full scan -API scans support full scanning, which can be enabled by using the `DAST_FULL_SCAN_ENABLED` environment variable. Domain validation isn't supported for full API scans. +API scans support full scanning, which can be enabled by using the `DAST_FULL_SCAN_ENABLED` +environment variable. Domain validation is not supported for full API scans. #### Host override -Specifications often define a host, which contains a domain name and a port. The host referenced may be different than the host of the API's review instance. -This can cause incorrect URLs to be imported, or a scan on an incorrect host. Use the `DAST_API_HOST_OVERRIDE` environment variable to override these values. +Specifications often define a host, which contains a domain name and a port. The +host referenced may be different than the host of the API's review instance. +This can cause incorrect URLs to be imported, or a scan on an incorrect host. +Use the `DAST_API_HOST_OVERRIDE` environment variable to override these values. For example, with a OpenAPI V3 specification containing: -```yml +```yaml servers: - url: https://api.host.com ``` -If the test version of the API is running at `https://api-test.host.com`, then the following DAST configuration can be used: +If the test version of the API is running at `https://api-test.host.com`, then +the following DAST configuration can be used: -```yml +```yaml include: - template: DAST.gitlab-ci.yml @@ -343,9 +375,11 @@ Note that `DAST_API_HOST_OVERRIDE` is only applied to specifications imported by #### Authentication using headers -Tokens in request headers are often used as a way to authenticate API requests. You can achieve this by using the `DAST_REQUEST_HEADERS` environment variable. Headers are applied to every request DAST makes. +Tokens in request headers are often used as a way to authenticate API requests. +You can achieve this by using the `DAST_REQUEST_HEADERS` environment variable. +Headers are applied to every request DAST makes. -```yml +```yaml include: - template: DAST.gitlab-ci.yml @@ -404,7 +438,8 @@ don't forget to add `stage: dast` when you override the template job definition. DAST can be [configured](#customizing-the-dast-settings) using environment variables. | Environment variable | Required | Description | -|-----------------------------| ----------|--------------------------------------------------------------------------------| +|-----------------------------| -----------|--------------------------------------------------------------------------------| +| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address from which to download the analyzer. | | `DAST_WEBSITE` | no| The URL of the website to scan. `DAST_API_SPECIFICATION` must be specified if this is omitted. | | `DAST_API_SPECIFICATION` | no | The API specification to import. `DAST_WEBSITE` must be specified if this is omitted. | | `DAST_AUTH_URL` | no | The authentication URL of the website to scan. Not supported for API scans. | @@ -416,14 +451,16 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia | `DAST_TARGET_AVAILABILITY_TIMEOUT` | no | Time limit in seconds to wait for target availability. Scan is attempted nevertheless if it runs out. Integer. Defaults to `60`. | | `DAST_FULL_SCAN_ENABLED` | no | Switches the tool to execute [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | | `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | no | Requires [domain validation](#domain-validation) when running DAST full scans. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. Not supported for API scans. | -| `DAST_AUTO_UPDATE_ADDONS` | no | Set to `false` to pin the versions of ZAProxy add-ons to those provided with the DAST image. Defaults to `true`. | +| `DAST_AUTO_UPDATE_ADDONS` | no | By default the versions of ZAP add-ons are pinned to those provided with the DAST image. Set to `true` to allow ZAP to download the latest versions. | | `DAST_API_HOST_OVERRIDE` | no | Used to override domains defined in API specification files. | | `DAST_EXCLUDE_RULES` | no | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from scans. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/master/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. | | `DAST_REQUEST_HEADERS` | no | Set to a comma-separated list of request header names and values. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | +| `DAST_ZAP_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | ### DAST command-line options -Not all DAST configuration is available via environment variables. To find out all possible options, run the following configuration. +Not all DAST configuration is available via environment variables. To find out all +possible options, run the following configuration. Available command-line options will be printed to the job log: ```yaml @@ -435,7 +472,8 @@ dast: - /analyze --help ``` -You must then overwrite the `script` command to pass in the appropriate argument. For example, AJAX spidering can be enabled by using `-j`, as shown in the following configuration: +You must then overwrite the `script` command to pass in the appropriate argument. +For example, debug messages can be enabled by using `-d`, as shown in the following configuration: ```yaml include: @@ -444,14 +482,16 @@ include: dast: script: - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -j -t $DAST_WEBSITE + - /analyze -d -t $DAST_WEBSITE ``` ### Custom ZAProxy configuration The ZAProxy server contains many [useful configurable values](https://gitlab.com/gitlab-org/gitlab/issues/36437#note_245801885). -Many key/values for `-config` remain undocumented, but there is an untested list of [possible keys](https://gitlab.com/gitlab-org/gitlab/issues/36437#note_244981023). -Note that these options are not supported by DAST, and may break the DAST scan when used. An example of how to rewrite the Authorization header value with `TOKEN` follows: +Many key/values for `-config` remain undocumented, but there is an untested list of +[possible keys](https://gitlab.com/gitlab-org/gitlab/issues/36437#note_244981023). +Note that these options are not supported by DAST, and may break the DAST scan +when used. An example of how to rewrite the Authorization header value with `TOKEN` follows: ```yaml include: @@ -479,50 +519,52 @@ successfully run. For more information, see [Offline environments](../offline_de To use DAST in an offline environment, you need: - GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). -- Docker Container Registry with a locally available copy of the DAST [container image](https://gitlab.com/gitlab-org/security-products/dast), found in the [DAST container registry](https://gitlab.com/gitlab-org/security-products/dast/container_registry). +- Docker Container Registry with a locally available copy of the DAST + [container image](https://gitlab.com/gitlab-org/security-products/dast), found in the + [DAST container registry](https://gitlab.com/gitlab-org/security-products/dast/container_registry). NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), -meaning the runner may try to pull remote images even if a local copy is available. Set GitLab -Runner's [`pull_policy` to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) -in an offline environment if you prefer using only locally available Docker images. +meaning the Runner tries to pull Docker images from the GitLab container registry even if a local +copy is available. GitLab Runner's [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) +in an offline environment if you prefer using only locally available Docker images. However, we +recommend keeping the pull policy setting to `always` if not in an offline environment, as this +enables the use of updated scanners in your CI/CD pipelines. ### Make GitLab DAST analyzer images available inside your Docker registry -For DAST, import the following default DAST analyzer image from `registry.gitlab.com` to your local "offline" -registry: +For DAST, import the following default DAST analyzer image from `registry.gitlab.com` to your [local Docker container registry](../../packages/container_registry/index.md): - `registry.gitlab.com/gitlab-org/security-products/dast:latest` The process for importing Docker images into a local offline Docker registry depends on **your network security policy**. Please consult your IT staff to find an accepted and approved -process by which external resources can be imported or temporarily accessed. Note that these scanners are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) -with new definitions, so consider if you are able to make periodic updates yourself. +process by which external resources can be imported or temporarily accessed. Note +that these scanners are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) +with new definitions, so consider if you're able to make periodic updates yourself. For details on saving and transporting Docker images as a file, see Docker's documentation on -[`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), -[`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). +[`docker save`](https://docs.docker.com/engine/reference/commandline/save/), +[`docker load`](https://docs.docker.com/engine/reference/commandline/load/), +[`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and +[`docker import`](https://docs.docker.com/engine/reference/commandline/import/). ### Set DAST CI job variables to use local DAST analyzers -1. Add the following configuration to your `.gitlab-ci.yml` file. You must replace `image` to refer - to the DAST Docker image hosted on your local Docker container registry: +Add the following configuration to your `.gitlab-ci.yml` file. You must replace `image` to refer to +the DAST Docker image hosted on your local Docker container registry: - ```yaml - include: - - template: DAST.gitlab-ci.yml - - dast: - image: registry.example.com/namespace/dast:latest - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -t $DAST_WEBSITE --auto-update-addons false -z"-silent" - ``` +```yaml +include: + - template: DAST.gitlab-ci.yml +dast: + image: registry.example.com/namespace/dast:latest +``` -The option `--auto-update-addons false` instructs ZAP not to update add-ons. +The DAST job should now use local copies of the DAST analyzers to scan your code and generate +security reports without requiring internet access. -The option `-z` passes the quoted `-silent` parameter to ZAP. The `-silent` parameter ensures ZAP -does not make any unsolicited requests including checking for updates. +Alternatively, you can use the variable `SECURE_ANALYZERS_PREFIX` to override the base registry address of the `dast` image. ## Reports @@ -530,7 +572,8 @@ The DAST job can emit various reports. ### List of URLs scanned -When DAST completes scanning, the merge request page states the number of URLs that were scanned. Click **View details** to view the web console output which includes the list of scanned URLs. +When DAST completes scanning, the merge request page states the number of URLs scanned. +Click **View details** to view the web console output which includes the list of scanned URLs.  @@ -539,9 +582,14 @@ When DAST completes scanning, the merge request page states the number of URLs t CAUTION: **Caution:** The JSON report artifacts are not a public API of DAST and their format is expected to change in the future. -The DAST tool always emits a JSON report file called `gl-dast-report.json` and sample reports can be found in the [DAST repository](https://gitlab.com/gitlab-org/security-products/dast/-/tree/master/test/end-to-end/expect). +The DAST tool always emits a JSON report file called `gl-dast-report.json` and +sample reports can be found in the +[DAST repository](https://gitlab.com/gitlab-org/security-products/dast/-/tree/master/test/end-to-end/expect). -There are two formats of data in the JSON report that are used side by side: the proprietary ZAP format which will be eventually deprecated, and a "common" format which will be the default in the future. +There are two formats of data in the JSON report that are used side by side: + +- The proprietary ZAP format that will be eventually deprecated. +- A common format that will be the default in the future. ### Other formats @@ -574,7 +622,9 @@ vulnerabilities in your groups, projects and pipelines. Read more about the ## Bleeding-edge vulnerability definitions -ZAProxy first creates rules in the `alpha` class. After a testing period with the community, they are promoted to `beta`. DAST uses `beta` definitions by default. To request `alpha` definitions, use `-a` as shown in the following configuration: +ZAProxy first creates rules in the `alpha` class. After a testing period with the +community, they are promoted to `beta`. DAST uses `beta` definitions by default. +To request `alpha` definitions, use `-a` as shown in the following configuration: ```yaml include: @@ -608,6 +658,18 @@ Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> +## Optimizing DAST + +By default, DAST will download all artifacts defined by previous jobs in the pipeline. If +your DAST job does not rely on `environment_url.txt` to define the URL under test or any other files created +in previous jobs, we recommend you don't download artifacts. To avoid downloading +artifacts, add the following to your `gitlab-ci.yml` file: + +```json +dast: + dependencies: [] +``` + ## Troubleshooting ### Running out of memory @@ -615,14 +677,14 @@ but commented out to help encourage others to add to it in the future. --> By default, ZAProxy, which DAST relies on, is allocated memory that sums to 25% of the total memory on the host. Since it keeps most of its information in memory during a scan, -it is possible for DAST to run out of memory while scanning large applications. +it's possible for DAST to run out of memory while scanning large applications. This results in the following error: ```plaintext [zap.out] java.lang.OutOfMemoryError: Java heap space ``` -Fortunately, it is straightforward to increase the amount of memory available +Fortunately, it's straightforward to increase the amount of memory available for DAST by overwriting the `script` key in the DAST template: ```yaml diff --git a/doc/user/application_security/dependency_scanning/analyzers.md b/doc/user/application_security/dependency_scanning/analyzers.md index 26352f21cfb..474f9339d0b 100644 --- a/doc/user/application_security/dependency_scanning/analyzers.md +++ b/doc/user/application_security/dependency_scanning/analyzers.md @@ -43,7 +43,7 @@ include: template: Dependency-Scanning.gitlab-ci.yml variables: - DS_ANALYZER_IMAGE_PREFIX: my-docker-registry/gl-images + SECURE_ANALYZERS_PREFIX: my-docker-registry/gl-images ``` This configuration requires that your custom registry provides images for all diff --git a/doc/user/application_security/dependency_scanning/img/dependency_scanning.png b/doc/user/application_security/dependency_scanning/img/dependency_scanning.png Binary files differdeleted file mode 100644 index 18df356f846..00000000000 --- a/doc/user/application_security/dependency_scanning/img/dependency_scanning.png +++ /dev/null diff --git a/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_0.png b/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_0.png Binary files differnew file mode 100644 index 00000000000..9f3990df957 --- /dev/null +++ b/doc/user/application_security/dependency_scanning/img/dependency_scanning_v13_0.png diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index cda621e61a6..53462cf232e 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -7,25 +7,24 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5105) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. Dependency Scanning helps to automatically find security vulnerabilities in your dependencies -while you are developing and testing your applications, for example when your +while you're developing and testing your applications, such as when your application is using an external (open source) library which is known to be vulnerable. ## Overview -If you are using [GitLab CI/CD](../../../ci/README.md), you can analyze your dependencies for known +If you're using [GitLab CI/CD](../../../ci/README.md), you can analyze your dependencies for known vulnerabilities using Dependency Scanning. All dependencies are scanned, including the transitive dependencies (also known as nested dependencies). - You can take advantage of Dependency Scanning by either [including the Dependency Scanning template](#configuration) in your existing `.gitlab-ci.yml` file or by implicitly using -[Auto Dependency Scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning-ultimate) -that is provided by [Auto DevOps](../../../topics/autodevops/index.md). +the [Auto Dependency Scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning-ultimate) +provided by [Auto DevOps](../../../topics/autodevops/index.md). GitLab checks the Dependency Scanning report, compares the found vulnerabilities between the source and target branches, and shows the information on the merge request. - + The results are sorted by the severity of the vulnerability: @@ -38,17 +37,16 @@ The results are sorted by the severity of the vulnerability: ## Requirements -To run a Dependency Scanning job, by default, you need GitLab Runner with the -[`docker`](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) or -[`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html#running-privileged-containers-for-the-runners) -executor running in privileged mode. If you're using the shared Runners on GitLab.com, -this is enabled by default. +To run Dependency Scanning jobs, by default, you need GitLab Runner with the +[`docker`](https://docs.gitlab.com/runner/executors/docker.html) or +[`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. +If you're using the shared Runners on GitLab.com, this is enabled by default. CAUTION: **Caution:** -If you use your own Runners, make sure that the Docker version you have installed +If you use your own Runners, make sure your installed version of Docker is **not** `19.03.0`. See [troubleshooting information](#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details. -Privileged mode is not necessary if you've [disabled Docker in Docker for Dependency Scanning](#disabling-docker-in-docker-for-dependency-scanning) +Beginning with GitLab 13.0, Docker privileged mode is necessary only if you've [enabled Docker-in-Docker for Dependency Scanning](#enabling-docker-in-docker). ## Supported languages and package managers @@ -56,16 +54,16 @@ The following languages and dependency managers are supported. | Language (package managers) | Supported | Scan tool(s) | |----------------------------- | --------- | ------------ | -| Java ([Gradle](https://gradle.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Java ([Maven](https://maven.apache.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| JavaScript ([npm](https://www.npmjs.com/), [yarn](https://classic.yarnpkg.com/en/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js/) | -| PHP ([Composer](https://getcomposer.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Python ([pip](https://pip.pypa.io/en/stable/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Java ([Gradle](https://gradle.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Java ([Maven](https://maven.apache.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| JavaScript ([npm](https://www.npmjs.com/), [yarn](https://classic.yarnpkg.com/en/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js/) | +| PHP ([Composer](https://getcomposer.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Python ([pip](https://pip.pypa.io/en/stable/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | | Python ([Pipfile](https://pipenv.kennethreitz.org/en/latest/basics/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/11756 "Pipfile.lock support for Dependency Scanning"))| not available | | Python ([poetry](https://python-poetry.org/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/7006 "Support Poetry in Dependency Scanning")) | not available | -| Ruby ([gem](https://rubygems.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) | -| Scala ([sbt](https://www.scala-sbt.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Go ([Go Modules](https://github.com/golang/go/wiki/Modules)) | yes ([alpha](https://gitlab.com/gitlab-org/gitlab/issues/7132)) | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Ruby ([gem](https://rubygems.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) | +| Scala ([sbt](https://www.scala-sbt.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Go ([Go Modules](https://github.com/golang/go/wiki/Modules)) | yes ([alpha](https://gitlab.com/gitlab-org/gitlab/issues/7132)) | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | ## Contribute your scanner @@ -73,7 +71,7 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Configuration -For GitLab 11.9 and later, to enable Dependency Scanning, you must +To enable Dependency Scanning for GitLab 11.9 and later, you must [include](../../../ci/yaml/README.md#includetemplate) the [`Dependency-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml) that's provided as a part of your GitLab installation. @@ -87,11 +85,10 @@ include: - template: Dependency-Scanning.gitlab-ci.yml ``` -The included template will create a `dependency_scanning` job in your CI/CD +The included template will create Dependency Scanning jobs in your CI/CD pipeline and scan your project's source code for possible vulnerabilities. - The results will be saved as a -[Dependency Scanning report artifact](../../../ci/yaml/README.md#artifactsreportsdependency_scanning-ultimate) +[Dependency Scanning report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsdependency_scanning-ultimate) that you can later download and analyze. Due to implementation limitations, we always take the latest Dependency Scanning artifact available. @@ -99,7 +96,6 @@ always take the latest Dependency Scanning artifact available. The Dependency Scanning settings can be changed through [environment variables](#available-variables) by using the [`variables`](../../../ci/yaml/README.md#variables) parameter in `.gitlab-ci.yml`. - For example: ```yaml @@ -113,23 +109,24 @@ variables: Because template is [evaluated before](../../../ci/yaml/README.md#include) the pipeline configuration, the last mention of the variable will take precedence. -### Overriding the Dependency Scanning template +### Overriding Dependency Scanning jobs CAUTION: **Deprecation:** Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic) is no longer supported. When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) instead. -If you want to override the job definition (for example, change properties like -`variables` or `dependencies`), you need to declare a `dependency_scanning` job -after the template inclusion and specify any additional keys under it. For example: +To override a job definition (for example, to change properties like `variables` or `dependencies`), +declare a new job with the same name as the one to override. Place this new job after the template +inclusion and specify any additional keys under it. For example, this disables `DS_REMEDIATE` for +the `gemnasium` analyzer: ```yaml include: - template: Dependency-Scanning.gitlab-ci.yml -dependency_scanning: +gemnasium-dependency_scanning: variables: - CI_DEBUG_TRACE: "true" + DS_REMEDIATE: "false" ``` ### Available variables @@ -143,19 +140,20 @@ The following variables allow configuration of global dependency scanning settin | Environment variable | Description | | --------------------------------------- |------------ | -| `DS_ANALYZER_IMAGE_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | +| `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | +| `DS_ANALYZER_IMAGE_PREFIX` | **DEPRECATED:** Use `SECURE_ANALYZERS_PREFIX` instead. | | `DS_DEFAULT_ANALYZERS` | Override the names of the official default images. Read more about [customizing analyzers](analyzers.md). | -| `DS_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#disabling-docker-in-docker-for-dependency-scanning).| +| `DS_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker). This variable is `true` by default. | | `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs to trust. | | `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. | #### Configuring Docker-in-Docker orchestrator -The following variables configure the Docker-in-Docker orchestrator. +The following variables configure the Docker-in-Docker orchestrator, and therefore are only used when the Docker-in-Docker mode is [enabled](#enabling-docker-in-docker). | Environment variable | Default | Description | | --------------------------------------- | ----------- | ----------- | -| `DS_ANALYZER_IMAGES` | | Comma separated list of custom images. The official default images are still enabled. Read more about [customizing analyzers](analyzers.md). | +| `DS_ANALYZER_IMAGES` | | Comma-separated list of custom images. The official default images are still enabled. Read more about [customizing analyzers](analyzers.md). | | `DS_ANALYZER_IMAGE_TAG` | | Override the Docker tag of the official default images. Read more about [customizing analyzers](analyzers.md). | | `DS_PULL_ANALYZER_IMAGES` | | Pull the images from the Docker registry (set to `0` to disable). | | `DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `300ms`, `1.5h`, or `2h45m`. | @@ -168,20 +166,20 @@ The following variables are used for configuring specific analyzers (used for a | Environment variable | Analyzer | Default | Description | | --------------------------------------- | ------------------ | ---------------------------- |------------ | -| `GEMNASIUM_DB_LOCAL_PATH` | `gemnasium` | `/gemnasium-db` | Path to local gemnasium database. | -| `GEMNASIUM_DB_REMOTE_URL` | `gemnasium` | `https://gitlab.com/gitlab-org/security-products/gemnasium-db.git` | Repository URL for fetching the gemnasium database. | +| `GEMNASIUM_DB_LOCAL_PATH` | `gemnasium` | `/gemnasium-db` | Path to local Gemnasium database. | +| `GEMNASIUM_DB_REMOTE_URL` | `gemnasium` | `https://gitlab.com/gitlab-org/security-products/gemnasium-db.git` | Repository URL for fetching the Gemnasium database. | | `GEMNASIUM_DB_REF_NAME` | `gemnasium` | `master` | Branch name for remote repository database. `GEMNASIUM_DB_REMOTE_URL` is required. | -| `DS_REMEDIATE` | `gemnasium` | `"true"` | Enable automatic remediation of vulnerable dependencies. | +| `DS_REMEDIATE` | `gemnasium` | `"true"` | Enable automatic remediation of vulnerable dependencies. | | `PIP_INDEX_URL` | `gemnasium-python` | `https://pypi.org/simple` | Base URL of Python Package Index. | -| `PIP_EXTRA_INDEX_URL` | `gemnasium-python` | | Array of [extra URLs](https://pip.pypa.io/en/stable/reference/pip_install/#cmdoption-extra-index-url) of package indexes to use in addition to `PIP_INDEX_URL`. Comma separated. | +| `PIP_EXTRA_INDEX_URL` | `gemnasium-python` | | Array of [extra URLs](https://pip.pypa.io/en/stable/reference/pip_install/#cmdoption-extra-index-url) of package indexes to use in addition to `PIP_INDEX_URL`. Comma-separated. | | `PIP_REQUIREMENTS_FILE` | `gemnasium-python` | | Pip requirements file to be scanned. | | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12412) in GitLab 12.2) | | `DS_PYTHON_VERSION` | `retire.js` | | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12296) in GitLab 12.1)| -| `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that will be passed to `maven` by the analyzer. See an example for [using private repos](../index.md#using-private-maven-repos). | -| `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that will be passed to `gradle` by the analyzer. | -| `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer will pass to `sbt`. | -| `BUNDLER_AUDIT_UPDATE_DISABLED` | `bundler-audit` | `"false"` | Disable automatic updates for the `bundler-audit` analyzer. Useful if you're running Dependency Scanning in an offline, air-gapped environment.| +| `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that will be passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repos). | +| `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that will be passed to `gradle` by the analyzer. | +| `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer will pass to `sbt`. | +| `BUNDLER_AUDIT_UPDATE_DISABLED` | `bundler-audit` | `"false"` | Disable automatic updates for the `bundler-audit` analyzer. Useful if you're running Dependency Scanning in an offline, air-gapped environment.| | `BUNDLER_AUDIT_ADVISORY_DB_URL` | `bundler-audit` | `https://github.com/rubysec/ruby-advisory-db` | URL of the advisory database used by bundler-audit. | | `BUNDLER_AUDIT_ADVISORY_DB_REF_NAME` | `bundler-audit` | `master` | Git ref for the advisory database specified by `BUNDLER_AUDIT_ADVISORY_DB_URL`. | | `RETIREJS_JS_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json` | Path or URL to `retire.js` JS vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` environment variable. | @@ -190,39 +188,31 @@ The following variables are used for configuring specific analyzers (used for a ### Using private Maven repos -If you have a private Maven repository which requires login credentials, +If your private Maven repository requires login credentials, you can use the `MAVEN_CLI_OPTS` environment variable. -Read more on [how to use private Maven repos](../index.md#using-private-maven-repos). +Read more on [how to use private Maven repositories](../index.md#using-private-maven-repos). -### Disabling Docker in Docker for Dependency Scanning +### Enabling Docker-in-Docker > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12487) in GitLab Ultimate 12.5. -You can avoid the need for Docker in Docker by running the individual analyzers. -This does not require running the executor in privileged mode. For example: +If needed, you can enable Docker-in-Docker to restore the Dependency Scanning behavior that existed +prior to GitLab 13.0. Follow these steps to do so: -```yaml -include: - - template: Dependency-Scanning.gitlab-ci.yml +1. Configure GitLab Runner with Docker-in-Docker in [privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). +1. Set the `DS_DISABLE_DIND` variable to `false`: -variables: - DS_DISABLE_DIND: "true" -``` + ```yaml + include: + - template: Dependency-Scanning.gitlab-ci.yml -This will create individual `<analyzer-name>-dependency_scanning` jobs for each analyzer that runs in your CI/CD pipeline. + variables: + DS_DISABLE_DIND: "false" + ``` -By removing Docker-in-Docker (DIND), GitLab relies on [Linguist](https://github.com/github/linguist) -to start relevant analyzers depending on the detected repository language(s) instead of the -[orchestrator](https://gitlab.com/gitlab-org/security-products/dependency-scanning/). However, there -are some differences in the way repository languages are detected between DIND and non-DIND. You can -observe these differences by checking both Linguist and the common library. For instance, Linguist -looks for `*.java` files to spin up the [gemnasium-maven](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven) -image, while orchestrator only looks for the existence of `pom.xml` or `build.gradle`. GitLab uses -Linguist to detect new file types in the default branch. This means that when introducing files or -dependencies for a new language or package manager, the corresponding scans won't be triggered in -the MR and will only run on the default branch once the MR is merged. This will be addressed by -[#211702](https://gitlab.com/gitlab-org/gitlab/-/issues/211702). +This creates a single `dependency_scanning` job in your CI/CD pipeline instead of multiple +`<analyzer-name>-dependency_scanning` jobs. ## Interacting with the vulnerabilities @@ -232,9 +222,8 @@ Once a vulnerability is found, you can interact with it. Read more on how to ## Solutions for vulnerabilities (auto-remediation) Some vulnerabilities can be fixed by applying the solution that GitLab -automatically generates. - -Read more about the [solutions for vulnerabilities](../index.md#solutions-for-vulnerabilities-auto-remediation). +automatically generates. Read more about the +[solutions for vulnerabilities](../index.md#solutions-for-vulnerabilities-auto-remediation). ## Security Dashboard @@ -371,33 +360,33 @@ it highlighted: CAUTION: **Deprecation:** Beginning with GitLab 12.9, dependency scanning no longer reports `undefined` severity and confidence levels. -Here is the description of the report file structure nodes and their meaning. All fields are mandatory to be present in -the report JSON unless stated otherwise. Presence of optional fields depends on the underlying analyzers being used. +This table describes the report file structure nodes and their meaning. All fields are mandatory to be present in +the report JSON, unless stated otherwise. The presence of optional fields depends on the underlying analyzers used. | Report JSON node | Description | |------------------------------------------------------|-------------| | `version` | Report syntax version used to generate this JSON. | | `vulnerabilities` | Array of vulnerability objects. | | `vulnerabilities[].id` | Unique identifier of the vulnerability. | -| `vulnerabilities[].category` | Where this vulnerability belongs (SAST, Dependency Scanning etc.). For Dependency Scanning, it will always be `dependency_scanning`. | -| `vulnerabilities[].name` | Name of the vulnerability, this must not include the occurrence's specific information. Optional. | -| `vulnerabilities[].message` | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. | +| `vulnerabilities[].category` | Where this vulnerability belongs, such as SAST or Dependency Scanning. For Dependency Scanning, it will always be `dependency_scanning`. | +| `vulnerabilities[].name` | Name of the vulnerability. Must not include the occurrence's specific information. Optional. | +| `vulnerabilities[].message` | A short text that describes the vulnerability. May include occurrence's specific information. Optional. | | `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | -| `vulnerabilities[].cve` | (**DEPRECATED - use `vulnerabilities[].id` instead**) A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | -| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | -| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | +| `vulnerabilities[].cve` | (**DEPRECATED - use `vulnerabilities[].id` instead**) A fingerprint string value that represents a concrete occurrence of the vulnerability. Used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | +| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | +| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | | `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | | `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | -| `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. | +| `vulnerabilities[].scanner.id` | ID of the scanner as a `snake_case` string. | | `vulnerabilities[].scanner.name` | Name of the scanner, for display purposes. | | `vulnerabilities[].location` | A node that tells where the vulnerability is located. | -| `vulnerabilities[].location.file` | Path to the dependencies file (e.g., `yarn.lock`). Optional. | +| `vulnerabilities[].location.file` | Path to the dependencies file (such as `yarn.lock`). Optional. | | `vulnerabilities[].location.dependency` | A node that describes the dependency of a project where the vulnerability is located. | | `vulnerabilities[].location.dependency.package` | A node that provides the information on the package where the vulnerability is located. | | `vulnerabilities[].location.dependency.package.name` | Name of the package where the vulnerability is located. Optional. | | `vulnerabilities[].location.dependency.version` | Version of the vulnerable package. Optional. | | `vulnerabilities[].identifiers` | An ordered array of references that identify a vulnerability on internal or external DBs. | -| `vulnerabilities[].identifiers[].type` | Type of the identifier. Possible values: common identifier types (among `cve`, `cwe`, `osvdb`, and `usn`) or analyzer-dependent ones (e.g. `gemnasium` for [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/)). | +| `vulnerabilities[].identifiers[].type` | Type of the identifier. Possible values: common identifier types (among `cve`, `cwe`, `osvdb`, and `usn`) or analyzer-dependent ones (such as `gemnasium` for [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/)). | | `vulnerabilities[].identifiers[].name` | Name of the identifier for display purpose. | | `vulnerabilities[].identifiers[].value` | Value of the identifier for matching purpose. | | `vulnerabilities[].identifiers[].url` | URL to identifier's documentation. Optional. | @@ -406,10 +395,10 @@ the report JSON unless stated otherwise. Presence of optional fields depends on | `vulnerabilities[].links[].url` | URL of the vulnerability details document. Optional. | | `remediations` | An array of objects containing information on cured vulnerabilities along with patch diffs to apply. Empty if no remediations provided by an underlying analyzer. | | `remediations[].fixes` | An array of strings that represent references to vulnerabilities fixed by this particular remediation. | -| `remediations[].fixes[].id` | The id of a fixed vulnerability. | +| `remediations[].fixes[].id` | The ID of a fixed vulnerability. | | `remediations[].fixes[].cve` | (**DEPRECATED - use `remediations[].fixes[].id` instead**) A string value that describes a fixed vulnerability in the same format as `vulnerabilities[].cve`. | | `remediations[].summary` | Overview of how the vulnerabilities have been fixed. | -| `remediations[].diff` | base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). | +| `remediations[].diff` | Base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). | ## Versioning and release process @@ -424,32 +413,33 @@ You can also [submit new vulnerabilities](https://gitlab.com/gitlab-org/security ## Running Dependency Scanning in an offline environment For self-managed GitLab instances in an environment with limited, restricted, or intermittent access -to external resources through the internet, some adjustments are required for dependency scannings jobs to run successfully. For more information, see [Offline environments](../offline_deployments/index.md). +to external resources through the internet, some adjustments are required for Dependency Scanning +jobs to run successfully. For more information, see [Offline environments](../offline_deployments/index.md). ### Requirements for offline Dependency Scanning Here are the requirements for using Dependency Scanning in an offline environment: -- [Disable Docker-In-Docker](#disabling-docker-in-docker-for-dependency-scanning) +- Keep Docker-In-Docker disabled (default). - GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). -- Docker Container Registry with locally available copies of dependency scanning [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. +- Docker Container Registry with locally available copies of Dependency Scanning [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. - Host an offline Git copy of the [gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/) - _Only if scanning Ruby projects_: Host an offline Git copy of the [advisory database](https://github.com/rubysec/ruby-advisory-db). - _Only if scanning npm/yarn projects_: Host an offline copy of the [retire.js](https://github.com/RetireJS/retire.js/) [node](https://github.com/RetireJS/retire.js/blob/master/repository/npmrepository.json) and [js](https://github.com/RetireJS/retire.js/blob/master/repository/jsrepository.json) advisory databases. NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), -meaning the runner will try to pull Docker images from the GitLab container registry even if a local +meaning the Runner tries to pull Docker images from the GitLab container registry even if a local copy is available. GitLab Runner's [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) in an offline environment if you prefer using only locally available Docker images. However, we -recommend keeping the pull policy setting to `always` as it will better enable updated scanners to -be utilized within your CI/CD pipelines. +recommend keeping the pull policy setting to `always` if not in an offline environment, as this +enables the use of updated scanners in your CI/CD pipelines. ### Make GitLab Dependency Scanning analyzer images available inside your Docker registry -For Dependency Scanning, import docker images ([supported languages and frameworks](#supported-languages-and-package-managers)) -from `registry.gitlab.com` to your offline docker registry. The Dependency Scanning analyzer -docker images are: +For Dependency Scanning with all [supported languages and frameworks](#supported-languages-and-package-managers), +import the following default Dependency Scanning analyzer images from `registry.gitlab.com` into +your [local Docker container registry](../../packages/container_registry/index.md): ```plaintext registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium:2 @@ -461,39 +451,34 @@ registry.gitlab.com/gitlab-org/security-products/analyzers/bundler-audit:2 The process for importing Docker images into a local offline Docker registry depends on **your network security policy**. Please consult your IT staff to find an accepted and approved -process by which external resources can be imported or temporarily accessed. Note that these scanners are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) -with new definitions, so consider if you are able to make periodic updates yourself. +process by which external resources can be imported or temporarily accessed. +Note that these scanners are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) +with new definitions, so consider if you can make periodic updates yourself. For details on saving and transporting Docker images as a file, see Docker's documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). -### Set Dependency Scanning CI config for "offline" use +### Set Dependency Scanning CI job variables to use local Dependency Scanning analyzers -Below is a general `.gitlab-ci.yml` template to configure your environment for running Dependency -Scanning offline: +Add the following configuration to your `.gitlab-ci.yml` file. You must replace +`DS_ANALYZER_IMAGE_PREFIX` to refer to your local Docker container registry: ```yaml include: - template: Dependency-Scanning.gitlab-ci.yml variables: - DS_DISABLE_DIND: "true" DS_ANALYZER_IMAGE_PREFIX: "docker-registry.example.com/analyzers" + GEMNASIUM_DB_REMOTE_URL: "gitlab.example.com/gemnasium-db.git" + GIT_SSL_NO_VERIFY: "true" ``` See explanations of the variables above in the [configuration section](#configuration). ### Specific settings for languages and package managers -For every language and package manager, add the following to the variables section of -`.gitlab-ci.yml`: - -```yaml -GEMNASIUM_DB_REMOTE_URL: "gitlab.example.com/gemnasium-db.git" -``` - -See the following sections for additional instructions on specific languages and package managers. +See the following sections for configuring specific languages and package managers. #### JavaScript (npm and yarn) projects @@ -515,10 +500,12 @@ BUNDLER_AUDIT_ADVISORY_DB_URL: "gitlab.example.com/ruby-advisory-db.git" #### Java (Maven) projects -When using a self-signed certificates, add the following to the variables section of`.gitlab-ci.yml`: +When using self-signed certificates, add the following job section to the `.gitlab-ci.yml`: ```yaml -MAVEN_CLI_OPTS="-Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true"` +gemnasium-maven-dependency_scanning: + variables: + MAVEN_CLI_OPTS: "-s settings.xml -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true" ``` #### Java (Gradle) projects @@ -527,13 +514,12 @@ When using self-signed certificates, add the following job section to the `.gitl ```yaml gemnasium-maven-dependency_scanning: - variables: - before_script: - - echo -n | openssl s_client -connect maven-repo.example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/internal.crt - - keytool -importcert -file /tmp/internal.crt -cacerts -storepass changeit -noprompt + before_script: + - echo -n | openssl s_client -connect maven-repo.example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/internal.crt + - keytool -importcert -file /tmp/internal.crt -cacerts -storepass changeit -noprompt ``` -This adds the self-signed certificates of your maven repository to the Java Key Store of the analyzer's docker image. +This adds the self-signed certificates of your Maven repository to the Java KeyStore of the analyzer's Docker image. #### Scala (sbt) projects @@ -541,41 +527,20 @@ When using self-signed certificates, add the following job section to the `.gitl ```yaml gemnasium-maven-dependency_scanning: - variables: - before_script: - - echo -n | openssl s_client -connect gitlab-airgap-test.us-west1-b.c.group-secure-a89fe7.internal:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/internal.crt - - keytool -importcert -file /tmp/internal.crt -cacerts -storepass changeit -noprompt -``` - -This adds the self-signed certificates of your maven repository to the Java Key Store of the analyzer's docker image. - -#### Python (pip) and Python (Pipfile) projects - -Add the following `pip.conf` to your repository to define your index URL and trust its self-signed -certificate: - -```toml -[global] -index-url = https://pypi.example.com -trusted-host = pypi.example.com -``` - -Add the following job section to `.gitlab-ci.yml`: - -```yaml -gemnasium-python-dependency_scanning: before_script: - - mkdir ~/.config/pip - - cp pip.conf ~/.config/pip/pip.conf + - echo -n | openssl s_client -connect maven-repo.example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/internal.crt + - keytool -importcert -file /tmp/internal.crt -cacerts -storepass changeit -noprompt ``` +This adds the self-signed certificates of your Maven repository to the Java KeyStore of the analyzer's Docker image. + #### Python (setuptools) -When using self-signed certificates for your private PyPi repo no extra job configuration (aside +When using self-signed certificates for your private PyPi repository, no extra job configuration (aside from the template `.gitlab-ci.yml` above) is needed. However, you must update your `setup.py` to -ensure that it can reach your private repo. Here is an example configuration: +ensure that it can reach your private repository. Here is an example configuration: -1. Update `setup.py` to create a `dependency_links` attribute pointing at your private repo for each +1. Update `setup.py` to create a `dependency_links` attribute pointing at your private repository for each dependency in the `install_requires` list: ```python @@ -596,11 +561,27 @@ ensure that it can reach your private repo. Here is an example configuration: setuptools.ssl_support.cert_paths = ['internal.crt'] ``` +## Limitations + +### Referencing local dependencies using a path in JavaScript projects + +The [Retire.js](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) analyzer +doesn't support dependency references made with [local paths](https://docs.npmjs.com/files/package.json#local-paths) +in the `package.json` of JavaScript projects. The dependency scan outputs the following error for +such references: + +```plaintext +ERROR: Could not find dependencies: <dependency-name>. You may need to run npm install +``` + +As a workaround, remove the [`retire.js`](analyzers.md#selecting-specific-analyzers) analyzer from +[DS_DEFAULT_ANALYZERS](#configuring-dependency-scanning). + ## Troubleshooting ### Error response from daemon: error processing tar file: docker-tar: relocation error -This error occurs when the Docker version used to run the SAST job is `19.03.0`. -You are advised to update to Docker `19.03.1` or greater. Older versions are not +This error occurs when the Docker version that runs the Dependency Scanning job is `19.03.00`. +Consider updating to Docker `19.03.1` or greater. Older versions are not affected. Read more in [this issue](https://gitlab.com/gitlab-org/gitlab/issues/13830#note_211354992 "Current SAST container fails"). diff --git a/doc/user/application_security/img/adding_a_dismissal_reason_v13_0.png b/doc/user/application_security/img/adding_a_dismissal_reason_v13_0.png Binary files differnew file mode 100644 index 00000000000..385236d08f2 --- /dev/null +++ b/doc/user/application_security/img/adding_a_dismissal_reason_v13_0.png diff --git a/doc/user/application_security/img/dismissed_info_v12_3.png b/doc/user/application_security/img/dismissed_info_v12_3.png Binary files differdeleted file mode 100644 index 92037493eaa..00000000000 --- a/doc/user/application_security/img/dismissed_info_v12_3.png +++ /dev/null diff --git a/doc/user/application_security/img/interacting_with_vulnerability_v13_0.png b/doc/user/application_security/img/interacting_with_vulnerability_v13_0.png Binary files differnew file mode 100644 index 00000000000..866ad74d42c --- /dev/null +++ b/doc/user/application_security/img/interacting_with_vulnerability_v13_0.png diff --git a/doc/user/application_security/img/interactive_reports.png b/doc/user/application_security/img/interactive_reports.png Binary files differdeleted file mode 100644 index 1b2ef0d3da9..00000000000 --- a/doc/user/application_security/img/interactive_reports.png +++ /dev/null diff --git a/doc/user/application_security/img/security_configuration_page_v12_9.png b/doc/user/application_security/img/security_configuration_page_v12_9.png Binary files differdeleted file mode 100644 index a81d82e03c3..00000000000 --- a/doc/user/application_security/img/security_configuration_page_v12_9.png +++ /dev/null diff --git a/doc/user/application_security/img/security_configuration_page_v13_1.png b/doc/user/application_security/img/security_configuration_page_v13_1.png Binary files differnew file mode 100644 index 00000000000..0147084f705 --- /dev/null +++ b/doc/user/application_security/img/security_configuration_page_v13_1.png diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index dadff8583db..eefa52eb5c3 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -44,6 +44,12 @@ To add Container Scanning, follow the steps listed in the [Container Scanning do To further configure any of the other scanners, refer to each scanner's documentation. +### Override the default registry base address + +By default, GitLab security scanners use `registry.gitlab.com/gitlab-org/security-products/analyzers` as the +base address for Docker images. You can override this globally by setting the variable +`SECURE_ANALYZERS_PREFIX` to another location. Note that this affects all scanners at once. + ## Security scanning tools GitLab uses the following tools to scan and report known vulnerabilities found in your project. @@ -54,6 +60,7 @@ GitLab uses the following tools to scan and report known vulnerabilities found i | [Dependency List](dependency_list/index.md) **(ULTIMATE)** | View your project's dependencies and their known vulnerabilities. | | [Dependency Scanning](dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. | | [Dynamic Application Security Testing (DAST)](dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities. | +| [Secret Detection](secret_detection/index.md) **(ULTIMATE)** | Analyze Git history for leaked secrets. | | [Security Dashboard](security_dashboard/index.md) **(ULTIMATE)** | View vulnerabilities in all your projects and groups. | | [Static Application Security Testing (SAST)](sast/index.md) **(ULTIMATE)** | Analyze source code for known vulnerabilities. | @@ -69,9 +76,10 @@ The scanning tools and vulnerabilities database are updated regularly. | [Static Application Security Testing (SAST)](sast/index.md) | Relies exclusively on [the tools GitLab wraps](sast/index.md#supported-languages-and-frameworks). The underlying analyzers are updated at least once per month if a relevant update is available. The vulnerabilities database is updated by the upstream tools. | Currently, you do not have to update GitLab to benefit from the latest vulnerabilities definitions. -The security tools are released as Docker images. The vendored job definitions to enable them use -the `x-y-stable` image tags that get overridden each time a new release of the tools is pushed. The -Docker images are updated to match the previous GitLab releases, so users automatically get the +The security tools are released as Docker images. The vendored job definitions that enable them use +major release tags according to [Semantic Versioning](https://semver.org/). Each new release of the +tools overrides these tags. +The Docker images are updated to match the previous GitLab releases, so users automatically get the latest versions of the scanning tools without having to do anything. There are some known issues with this approach, however, and there is a [plan to resolve them](https://gitlab.com/gitlab-org/gitlab/issues/9725). @@ -80,9 +88,6 @@ with this approach, however, and there is a > Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.8. -CAUTION: **Warning:** -This feature is currently [Alpha](https://about.gitlab.com/handbook/product/#alpha-beta-ga) and while you can start using it, it may receive important changes in the future. - Each security vulnerability in the merge request report or the [Security Dashboard](security_dashboard/index.md) is actionable. Click an entry to view detailed information with several options: @@ -95,25 +100,27 @@ information with several options: - [Solution](#solutions-for-vulnerabilities-auto-remediation): For some vulnerabilities, a solution is provided for how to fix the vulnerability. - + ### Dismissing a vulnerability -You can dismiss vulnerabilities by clicking the **Dismiss vulnerability** button. -This will dismiss the vulnerability and re-render it to reflect its dismissed state. -If you wish to undo this dismissal, you can click the **Undo dismiss** button. +To dismiss a vulnerability, you must set its status to Dismissed. Follow these steps to do so: + +1. Select the vulnerability in the Security Dashboard. +1. Select **Dismissed** from the **Status** selector menu at the top-right. + +You can undo this action by selecting a different status from the same menu. #### Adding a dismissal reason > Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0. -When dismissing a vulnerability, it's often helpful to provide a reason for doing so. -If you press the comment button next to **Dismiss vulnerability** in the modal, -a text box appears for you to add a comment with your dismissal. -Once added, you can edit or delete it. This allows you to add and update -context for a vulnerability as you learn more over time. +When dismissing a vulnerability, it's often helpful to provide a reason for doing so. Upon setting a +vulnerability's status to Dismissed, a text box appears for you to add a comment with your +dismissal. Once added, you can edit or delete it. This allows you to add and update context for a +vulnerability as you learn more over time. - + #### Dismissing multiple vulnerabilities @@ -193,9 +200,19 @@ security team when a merge request would introduce one of the following security - A security vulnerability - A software license compliance violation -This threshold is defined as `high`, `critical`, or `unknown` severity. When any vulnerabilities are -present within a merge request, an approval is required from the `Vulnerability-Check` approver -group. +The security vulnerability threshold is defined as `high`, `critical`, or `unknown` severity. The +`Vulnerability-Check` approver group must approve merge requests that contain vulnerabilities. + +When GitLab can assess vulnerability severity, the rating can be one of the following: + +- `unknown` +- `low` +- `medium` +- `high` +- `critical` + +The rating `unknown` indicates that the underlying scanner doesn't contain or provide a severity +rating. ### Enabling Security Approvals within a project @@ -209,7 +226,7 @@ Any code changes cause the approvals required to reset. An approval is required when a security report: -- Contains a new vulnerability of `high`, `critical`, or `unknown` severity. +- Contains a new vulnerability of `high`, `critical`, or `unknown` severity, regardless of dismissal. - Is not generated during pipeline execution. An approval is optional when a security report: @@ -259,7 +276,7 @@ to pass a username and password. You can set it under your project's settings so that your credentials aren't exposed in `.gitlab-ci.yml`. If the username is `myuser` and the password is `verysecret` then you would -[set the following variable](../../ci/variables/README.md#via-the-ui) +[set the following variable](../../ci/variables/README.md#create-a-custom-variable-in-the-ui) under your project's settings: | Type | Key | Value | @@ -313,7 +330,8 @@ You can do it quickly by following the hyperlink given to run a new pipeline. ### Getting error message `sast job: stage parameter should be [some stage name here]` -When including a security job template like [`SAST`](sast/index.md#configuration), +When [including](../../ci/yaml/README.md#includetemplate) a `.gitlab-ci.yml` template +like [`SAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml), the following error may occur, depending on your GitLab CI/CD configuration: ```plaintext @@ -326,15 +344,115 @@ This error appears when the included job's stage (named `test`) isn't declared i To fix this issue, you can either: - Add a `test` stage in your `.gitlab-ci.yml`. -- Change the default stage of the included security jobs. For example, with `SAST`: +- Change the default stage of the included security jobs. For example, with SpotBugs (SAST): ```yaml include: template: SAST.gitlab-ci.yml - sast: + spotbugs-sast: stage: unit-tests ``` -[Learn more on overriding the SAST template](sast/index.md#overriding-the-sast-template). +[Learn more on overriding SAST jobs](sast/index.md#overriding-sast-jobs). All the security scanning tools define their stage, so this error can occur with all of them. + +### Getting error message `sast job: config key may not be used with 'rules': only/except` + +When [including](../../ci/yaml/README.md#includetemplate) a `.gitlab-ci.yml` template +like [`SAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml), +the following error may occur, depending on your GitLab CI/CD configuration: + +```plaintext +Found errors in your .gitlab-ci.yml: + + jobs:sast config key may not be used with `rules`: only/except +``` + +This error appears when the included job's `rules` configuration has been [overridden](sast/index.md#overriding-sast-jobs) +with [the deprecated `only` or `except` syntax.](../../ci/yaml/README.md#onlyexcept-basic) +To fix this issue, you must either: + +- [Transition your `only/except` syntax to `rules`](#transitioning-your-onlyexcept-syntax-to-rules). +- (Temporarily) [Pin your templates to the deprecated versions](#pin-your-templates-to-the-deprecated-versions) + +[Learn more on overriding SAST jobs](sast/index.md#overriding-sast-jobs). + +#### Transitioning your `only/except` syntax to `rules` + +When overriding the template to control job execution, previous instances of +[`only` or `except`](../../ci/yaml/README.md#onlyexcept-basic) are no longer compatible +and must be transitioned to [the `rules` syntax](../../ci/yaml/README.md#rules). + +If your override is aimed at limiting jobs to only run on `master`, the previous syntax +would look similar to: + +```yaml +include: + - template: SAST.gitlab-ci.yml + +# Ensure that the scanning is only executed on master or merge requests +spotbugs-sast: + only: + refs: + - master + - merge_requests +``` + +To transition the above configuration to the new `rules` syntax, the override +would be written as follows: + +```yaml +include: + - template: SAST.gitlab-ci.yml + +# Ensure that the scanning is only executed on master or merge requests +spotbugs-sast: + rules: + - if: $CI_COMMIT_BRANCH == "master" + - if: $CI_MERGE_REQUEST_ID +``` + +If your override is aimed at limiting jobs to only run on branches, not tags, +it would look similar to: + +```yaml +include: + - template: SAST.gitlab-ci.yml + +# Ensure that the scanning is not executed on tags +spotbugs-sast: + except: + - tags +``` + +To transition to the new `rules` syntax, the override would be rewritten as: + +```yaml +include: + - template: SAST.gitlab-ci.yml + +# Ensure that the scanning is not executed on tags +spotbugs-sast: + rules: + - if: $CI_COMMIT_TAG == null +``` + +[Learn more on the usage of `rules`](../../ci/yaml/README.md#rules). + +#### Pin your templates to the deprecated versions + +To ensure the latest support, we **strongly** recommend that you migrate to [`rules`](../../ci/yaml/README.md#rules). + +If you're unable to immediately update your CI configuration, there are several workarounds that +involve pinning to the previous template versions, for example: + + ```yaml + include: + remote: 'https://gitlab.com/gitlab-org/gitlab/-/raw/12-10-stable-ee/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml' + ``` + +Additionally, we provide a dedicated project containing the versioned legacy templates. +This can be useful for offline setups or anyone wishing to use [Auto DevOps](../../topics/autodevops/index.md). + +Instructions are available in the [legacy template project](https://gitlab.com/gitlab-org/auto-devops-v12-10). diff --git a/doc/user/application_security/offline_deployments/index.md b/doc/user/application_security/offline_deployments/index.md index 61b34901849..3deb38902f2 100644 --- a/doc/user/application_security/offline_deployments/index.md +++ b/doc/user/application_security/offline_deployments/index.md @@ -61,6 +61,14 @@ Once a vulnerability is found, you can interact with it. Read more on how to Please note that in some cases the reported vulnerabilities provide metadata that can contain external links exposed in the UI. These links might not be accessible within an offline environment. +### Suggested Solutions for vulnerabilities + +The [suggested solutions](../index.md#solutions-for-vulnerabilities-auto-remediation) feature +(auto-remediation) is available for Dependency Scanning and Container Scanning, but may not work +depending on your instance's configuration. We can only suggest solutions, which are generally more +current versions that have been patched, when we are able to access up-to-date registry services +hosting the latest versions of that dependency or image. + ### Scanner signature and rule updates When connected to the internet, some scanners will reference public databases @@ -79,3 +87,4 @@ above. You can find more information at each of the pages below: - [SAST offline directions](../sast/index.md#running-sast-in-an-offline-environment) - [DAST offline directions](../dast/index.md#running-dast-in-an-offline-environment) - [License Compliance offline directions](../../compliance/license_compliance/index.md#running-license-compliance-in-an-offline-environment) +- [Dependency Scanning offline directions](../dependency_scanning/index.md#running-dependency-scanning-in-an-offline-environment) diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index 9010a7cae0b..08078a66719 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -52,7 +52,7 @@ include: - template: SAST.gitlab-ci.yml variables: - SAST_ANALYZER_IMAGE_PREFIX: my-docker-registry/gl-images + SECURE_ANALYZERS_PREFIX: my-docker-registry/gl-images ``` This configuration requires that your custom registry provides images for all @@ -149,7 +149,7 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) | End line | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | | Start column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | | End column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | -| External id (e.g. CVE) | 𐄂 | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| External ID (e.g. CVE) | 𐄂 | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | | URLs | ✓ | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | | Internal doc/explanation | ✓ | ⚠ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | 𐄂 | | Solution | ✓ | 𐄂 | 𐄂 | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | diff --git a/doc/user/application_security/sast/img/sast_v12_9.png b/doc/user/application_security/sast/img/sast_v12_9.png Binary files differdeleted file mode 100644 index 3c6ee7a276b..00000000000 --- a/doc/user/application_security/sast/img/sast_v12_9.png +++ /dev/null diff --git a/doc/user/application_security/sast/img/sast_v13_0.png b/doc/user/application_security/sast/img/sast_v13_0.png Binary files differnew file mode 100644 index 00000000000..b4aea6ea466 --- /dev/null +++ b/doc/user/application_security/sast/img/sast_v13_0.png diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index a6457d58fe2..370c6d0e8e7 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -13,7 +13,7 @@ to learn how to protect your organization. ## Overview -If you are using [GitLab CI/CD](../../../ci/README.md), you can analyze your source code for known +If you're using [GitLab CI/CD](../../../ci/README.md), you can analyze your source code for known vulnerabilities using Static Application Security Testing (SAST). You can take advantage of SAST by doing one of the following: @@ -25,7 +25,7 @@ You can take advantage of SAST by doing one of the following: GitLab checks the SAST report, compares the found vulnerabilities between the source and target branches, and shows the information right on the merge request. - + The results are sorted by the priority of the vulnerability: @@ -36,6 +36,9 @@ The results are sorted by the priority of the vulnerability: 1. Unknown 1. Everything else +NOTE: **Note:** +A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard won't show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard won't show SAST results. The analyzer will output an [exit code](../../../development/integrations/secure.md#exit-code) on failure. + ## Use cases - Your code has a potentially dangerous attribute in a class, or unsafe code @@ -45,19 +48,17 @@ The results are sorted by the priority of the vulnerability: ## Requirements -To run a SAST job, by default, you need GitLab Runner with the -[`docker`](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) or -[`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html#running-privileged-containers-for-the-runners) -executor running in privileged mode. If you're using the shared Runners on GitLab.com, -this is enabled by default. +To run SAST jobs, by default, you need GitLab Runner with the +[`docker`](https://docs.gitlab.com/runner/executors/docker.html) or +[`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. +If you're using the shared Runners on GitLab.com, this is enabled by default. -Privileged mode is not necessary if you've [disabled Docker in Docker -for SAST](#disabling-docker-in-docker-for-sast) +Beginning with GitLab 13.0, Docker privileged mode is necessary only if you've [enabled Docker-in-Docker for SAST](#enabling-docker-in-docker). CAUTION: **Caution:** Our SAST jobs currently expect a Linux container type. Windows containers are not yet supported. CAUTION: **Caution:** -If you use your own Runners, make sure that the Docker version you have installed +If you use your own Runners, make sure the Docker version installed is **not** `19.03.0`. See [troubleshooting information](#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details. ## Supported languages and frameworks @@ -66,9 +67,10 @@ The following table shows which languages, package managers and frameworks are s | Language (package managers) / framework | Scan tool | Introduced in GitLab Version | |-----------------------------------------------------------------------------|----------------------------------------------------------------------------------------|------------------------------| -| .NET | [Security Code Scan](https://security-code-scan.github.io) | 11.0 | +| .NET Core | [Security Code Scan](https://security-code-scan.github.io) | 11.0 | +| .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0 | | Any | [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) | 11.9 | -| Apex (Salesforce) | [pmd](https://pmd.github.io/pmd/index.html) | 12.1 | +| Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1 | | C/C++ | [Flawfinder](https://dwheeler.com/flawfinder/) | 10.7 | | Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.10 | | Go | [Gosec](https://github.com/securego/gosec) | 10.7 | @@ -82,7 +84,7 @@ The following table shows which languages, package managers and frameworks are s | React | [ESLint react plugin](https://github.com/yannickcr/eslint-plugin-react) | 12.5 | | Ruby on Rails | [brakeman](https://brakemanscanner.org) | 10.3 | | Scala ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.0 (SBT) & 11.9 (Ant, Gradle, Maven) | -| TypeScript | [TSLint config security](https://github.com/webschik/tslint-config-security/) | 11.9 | +| TypeScript | [`tslint-config-security`](https://github.com/webschik/tslint-config-security/) | 11.9 | NOTE: **Note:** The Java analyzers can also be used for variants like the @@ -101,7 +103,7 @@ provided by [Auto DevOps](../../../topics/autodevops/index.md). For GitLab 11.9 and later, to enable SAST you must [include](../../../ci/yaml/README.md#includetemplate) the [`SAST.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) -that is provided as a part of your GitLab installation. For GitLab versions earlier than 11.9, you +provided as a part of your GitLab installation. For GitLab versions earlier than 11.9, you can copy and use the job as defined that template. Add the following to your `.gitlab-ci.yml` file: @@ -111,22 +113,19 @@ include: - template: SAST.gitlab-ci.yml ``` -The included template will create a `sast` job in your CI/CD pipeline and scan +The included template will create SAST jobs in your CI/CD pipeline and scan your project's source code for possible vulnerabilities. The results will be saved as a -[SAST report artifact](../../../ci/yaml/README.md#artifactsreportssast-ultimate) +[SAST report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportssast-ultimate) that you can later download and analyze. Due to implementation limitations, we -always take the latest SAST artifact available. Behind the scenes, the -[GitLab SAST Docker image](https://gitlab.com/gitlab-org/security-products/sast) -is used to detect the languages/frameworks and in turn runs the matching scan tools. +always take the latest SAST artifact available. ### Customizing the SAST settings The SAST settings can be changed through [environment variables](#available-variables) by using the [`variables`](../../../ci/yaml/README.md#variables) parameter in `.gitlab-ci.yml`. - In the following example, we include the SAST template and at the same time we set the `SAST_GOSEC_LEVEL` variable to `2`: @@ -139,25 +138,26 @@ variables: ``` Because the template is [evaluated before](../../../ci/yaml/README.md#include) -the pipeline configuration, the last mention of the variable will take precedence. +the pipeline configuration, the last mention of the variable takes precedence. -### Overriding the SAST template +### Overriding SAST jobs CAUTION: **Deprecation:** Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic) is no longer supported. When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) instead. -If you want to override the job definition (for example, change properties like -`variables` or `dependencies`), you need to declare a `sast` job after the -template inclusion and specify any additional keys under it. For example: +To override a job definition, (for example, change properties like `variables` or `dependencies`), +declare a job with the same name as the SAST job to override. Place this new job after the template +inclusion and specify any additional keys under it. For example, this enables `FAIL_NEVER` for the +`spotbugs` analyzer: ```yaml include: - template: SAST.gitlab-ci.yml -sast: +spotbugs-sast: variables: - CI_DEBUG_TRACE: "true" + FAIL_NEVER: 1 ``` ### Using environment variables to pass credentials for private repositories @@ -170,57 +170,42 @@ it via [custom environment variables](#custom-environment-variables). #### Using a variable to pass username and password to a private Maven repository -If you have a private Maven repository which requires login credentials, +If your private Maven repository requires login credentials, you can use the `MAVEN_CLI_OPTS` environment variable. -Read more on [how to use private Maven repos](../index.md#using-private-maven-repos). +Read more on [how to use private Maven repositories](../index.md#using-private-maven-repos). -### Disabling Docker in Docker for SAST +### Enabling Docker-in-Docker -You can avoid the need for Docker in Docker by running the individual analyzers. -This does not require running the executor in privileged mode. For example: +If needed, you can enable Docker-in-Docker to restore the SAST behavior that existed prior to GitLab +13.0. Follow these steps to do so: -```yaml -include: - - template: SAST.gitlab-ci.yml +1. Configure GitLab Runner with Docker-inDocker in [privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). +1. Set the variable `SAST_DISABLE_DIND` set to `false`: -variables: - SAST_DISABLE_DIND: "true" -``` + ```yaml + include: + - template: SAST.gitlab-ci.yml -This will create individual `<analyzer-name>-sast` jobs for each analyzer that runs in your CI/CD pipeline. + variables: + SAST_DISABLE_DIND: "false" + ``` -By removing Docker-in-Docker (DIND), GitLab relies on [Linguist](https://github.com/github/linguist) -to start relevant analyzers depending on the detected repository language(s) instead of the -[orchestrator](https://gitlab.com/gitlab-org/security-products/dependency-scanning/). However, there -are some differences in the way repository languages are detected between DIND and non-DIND. You can -observe these differences by checking both Linguist and the common library. For instance, Linguist -looks for `*.java` files to spin up the [spotbugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) -image, while orchestrator only looks for the existence of `pom.xml`, `build.xml`, `gradlew`, -`grailsw`, or `mvnw`. GitLab uses Linguist to detect new file types in the default branch. This -means that when introducing files or dependencies for a new language or package manager, the -corresponding scans won't be triggered in the MR and will only run on the default branch once the -MR is merged. This will be addressed by [#211702](https://gitlab.com/gitlab-org/gitlab/-/issues/211702). - -NOTE: **Note:** -With the current language detection logic, any new languages or frameworks introduced within the -context of a merge request don't trigger a corresponding scan. These scans only occur once the code -is committed to the default branch. +This creates a single `sast` job in your CI/CD pipeline instead of multiple `<analyzer-name>-sast` +jobs. -#### Enabling kubesec analyzer +#### Enabling Kubesec analyzer > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12752) in GitLab Ultimate 12.6. -When [Docker in Docker is disabled](#disabling-docker-in-docker-for-sast), -you will need to set `SCAN_KUBERNETES_MANIFESTS` to `"true"` to enable the -kubesec analyzer. In `.gitlab-ci.yml`, define: +You need to set `SCAN_KUBERNETES_MANIFESTS` to `"true"` to enable the +Kubesec analyzer. In `.gitlab-ci.yml`, define: ```yaml include: - template: SAST.gitlab-ci.yml variables: - SAST_DISABLE_DIND: "true" SCAN_KUBERNETES_MANIFESTS: "true" ``` @@ -246,9 +231,6 @@ stages: include: - template: SAST.gitlab-ci.yml -variables: - SAST_DISABLE_DIND: "true" - build: stage: build script: @@ -291,10 +273,11 @@ The following are Docker image-related variables. | Environment variable | Description | |------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAST_ANALYZER_IMAGE_PREFIX` | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). | -| `SAST_ANALYZER_IMAGE_TAG` | **DEPRECATED:** Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | +| `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). | +| `SAST_ANALYZER_IMAGE_PREFIX` | **DEPRECATED**: Use `SECURE_ANALYZERS_PREFIX` instead. | +| `SAST_ANALYZER_IMAGE_TAG` | **DEPRECATED:** Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | | `SAST_DEFAULT_ANALYZERS` | Override the names of default images. Read more about [customizing analyzers](analyzers.md). | -| `SAST_DISABLE_DIND` | Disable Docker in Docker and run analyzers [individually](#disabling-docker-in-docker-for-sast). | +| `SAST_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker). This variable is `true` by default. | #### Vulnerability filters @@ -307,25 +290,22 @@ Some analyzers make it possible to filter out vulnerabilities under a given thre | `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | | `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | | `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | -| `SAST_GOSEC_LEVEL` | 0 | Ignore gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 2=Medium, 3=High. | -| `SAST_GITLEAKS_COMMIT_FROM` | - | The commit a gitleaks scan starts at. | -| `SAST_GITLEAKS_COMMIT_TO` | - | The commit a gitleaks scan ends at. | -| `SAST_GITLEAKS_HISTORIC_SCAN` | false | Flag to enable a historic gitleaks scan. | +| `SAST_GOSEC_LEVEL` | 0 | Ignore Gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 2=Medium, 3=High. | +| `SAST_GITLEAKS_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. | +| `SAST_GITLEAKS_COMMIT_TO` | - | The commit a Gitleaks scan ends at. | +| `SAST_GITLEAKS_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | #### Docker-in-Docker orchestrator -The following variables configure the Docker-in-Docker orchestrator. +The following variables configure the Docker-in-Docker orchestrator, and therefore are only used when the Docker-in-Docker mode is [enabled](#enabling-docker-in-docker). | Environment variable | Default value | Description | |------------------------------------------|---------------|-------------| -| `SAST_ANALYZER_IMAGES` | | Comma-separated list of custom images. Default images are still enabled. Read more about [customizing analyzers](analyzers.md). Not available when [Docker-in-Docker is disabled](#disabling-docker-in-docker-for-sast). | -| `SAST_PULL_ANALYZER_IMAGES` | 1 | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md). Not available when [Docker-in-Docker is disabled](#disabling-docker-in-docker-for-sast). | -| `SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | -| `SAST_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | -| `SAST_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m".| - -NOTE: **Note:** -Timeout variables are not applicable for setups with [disabled Docker In Docker](index.md#disabling-docker-in-docker-for-sast). +| `SAST_ANALYZER_IMAGES` | | Comma-separated list of custom images. Default images are still enabled. Read more about [customizing analyzers](analyzers.md). | +| `SAST_PULL_ANALYZER_IMAGES` | 1 | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md). | +| `SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`. | +| `SAST_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`. | +| `SAST_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h` or `2h45m`.| #### Analyzer settings @@ -333,25 +313,26 @@ Some analyzers can be customized with environment variables. | Environment variable | Analyzer | Description | |-----------------------------|----------|-------------| -| `SCAN_KUBERNETES_MANIFESTS` | kubesec | Set to `"true"` to scan Kubernetes manifests when [Docker in Docker](#disabling-docker-in-docker-for-sast) is disabled. | -| `ANT_HOME` | spotbugs | The `ANT_HOME` environment variable. | -| `ANT_PATH` | spotbugs | Path to the `ant` executable. | -| `GRADLE_PATH` | spotbugs | Path to the `gradle` executable. | -| `JAVA_OPTS` | spotbugs | Additional arguments for the `java` executable. | -| `JAVA_PATH` | spotbugs | Path to the `java` executable. | -| `SAST_JAVA_VERSION` | spotbugs | Which Java version to use. Supported versions are `8` and `11`. Defaults to `8`. | -| `MAVEN_CLI_OPTS` | spotbugs | Additional arguments for the `mvn` or `mvnw` executable. | -| `MAVEN_PATH` | spotbugs | Path to the `mvn` executable. | -| `MAVEN_REPO_PATH` | spotbugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | -| `SBT_PATH` | spotbugs | Path to the `sbt` executable. | -| `FAIL_NEVER` | spotbugs | Set to `1` to ignore compilation failure. | +| `SCAN_KUBERNETES_MANIFESTS` | Kubesec | Set to `"true"` to scan Kubernetes manifests. | +| `ANT_HOME` | SpotBugs | The `ANT_HOME` environment variable. | +| `ANT_PATH` | SpotBugs | Path to the `ant` executable. | +| `GRADLE_PATH` | SpotBugs | Path to the `gradle` executable. | +| `JAVA_OPTS` | SpotBugs | Additional arguments for the `java` executable. | +| `JAVA_PATH` | SpotBugs | Path to the `java` executable. | +| `SAST_JAVA_VERSION` | SpotBugs | Which Java version to use. Supported versions are `8` and `11`. Defaults to `8`. | +| `MAVEN_CLI_OPTS` | SpotBugs | Additional arguments for the `mvn` or `mvnw` executable. | +| `MAVEN_PATH` | SpotBugs | Path to the `mvn` executable. | +| `MAVEN_REPO_PATH` | SpotBugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | +| `SBT_PATH` | SpotBugs | Path to the `sbt` executable. | +| `FAIL_NEVER` | SpotBugs | Set to `1` to ignore compilation failure. | +| `SAST_GOSEC_CONFIG` | Gosec | Path to configuration for Gosec (optional). | #### Custom environment variables > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18193) in GitLab Ultimate 12.5. In addition to the aforementioned SAST configuration variables, -all [custom environment variables](../../../ci/variables/README.md#creating-a-custom-environment-variable) are propagated +all [custom environment variables](../../../ci/variables/README.md#custom-environment-variables) are propagated to the underlying SAST analyzer images if [the SAST vendored template](#configuration) is used. @@ -451,16 +432,16 @@ the report JSON unless stated otherwise. Presence of optional fields depends on | `version` | Report syntax version used to generate this JSON. | | `vulnerabilities` | Array of vulnerability objects. | | `vulnerabilities[].id` | Unique identifier of the vulnerability. | -| `vulnerabilities[].category` | Where this vulnerability belongs (SAST, Dependency Scanning etc.). For SAST, it will always be `sast`. | -| `vulnerabilities[].name` | Name of the vulnerability, this must not include the occurrence's specific information. Optional. | +| `vulnerabilities[].category` | Where this vulnerability belongs (such as SAST, Dependency Scanning). For SAST, it will always be `sast`. | +| `vulnerabilities[].name` | Name of the vulnerability. Must not include the occurrence's specific information. Optional. | | `vulnerabilities[].message` | A short text that describes the vulnerability, it may include the occurrence's specific information. Optional. | | `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | | `vulnerabilities[].cve` | (**DEPRECATED - use `vulnerabilities[].id` instead**) A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | -| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | -| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | +| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | +| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | | `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | | `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | -| `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. | +| `vulnerabilities[].scanner.id` | ID of the scanner as a snake_case string. | | `vulnerabilities[].scanner.name` | Name of the scanner, for display purposes. | | `vulnerabilities[].location` | A node that tells where the vulnerability is located. | | `vulnerabilities[].location.file` | Path to the file where the vulnerability is located. Optional. | @@ -468,29 +449,15 @@ the report JSON unless stated otherwise. Presence of optional fields depends on | `vulnerabilities[].location.end_line` | The last line of the code affected by the vulnerability. Optional. | | `vulnerabilities[].location.class` | If specified, provides the name of the class where the vulnerability is located. Optional. | | `vulnerabilities[].location.method` | If specified, provides the name of the method where the vulnerability is located. Optional. | -| `vulnerabilities[].identifiers` | An ordered array of references that identify a vulnerability on internal or external DBs. | -| `vulnerabilities[].identifiers[].type` | Type of the identifier. Possible values: common identifier types (among `cve`, `cwe`, `osvdb`, and `usn`) or analyzer-dependent ones (e.g., `bandit_test_id` for [Bandit analyzer](https://wiki.openstack.org/wiki/Security/Projects/Bandit)). | +| `vulnerabilities[].identifiers` | An ordered array of references that identify a vulnerability on internal or external databases. | +| `vulnerabilities[].identifiers[].type` | Type of the identifier. Possible values: common identifier types (among `cve`, `cwe`, `osvdb`, and `usn`) or analyzer-dependent ones (like `bandit_test_id` for [Bandit analyzer](https://wiki.openstack.org/wiki/Security/Projects/Bandit)). | | `vulnerabilities[].identifiers[].name` | Name of the identifier for display purposes. | | `vulnerabilities[].identifiers[].value` | Value of the identifier for matching purposes. | | `vulnerabilities[].identifiers[].url` | URL to identifier's documentation. Optional. | ## Secret detection -GitLab is also able to detect secrets and credentials that have been unintentionally pushed to the -repository (for example, an API key that allows write access to third-party deployment -environments). - -This check is performed by a specific analyzer during the `sast` job. It runs regardless of the programming -language of your app, and you don't need to change anything to your -CI/CD configuration file to turn it on. Results are available in the SAST report. - -GitLab currently includes [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) checks. - -NOTE: **Note:** -The secrets analyzer will ignore "Password in URL" vulnerabilities if the password begins -with a dollar sign (`$`) as this likely indicates the password being used is an environment -variable. For example, `https://username:$password@example.com/path/to/repo` will not be -detected, whereas `https://username:password@example.com/path/to/repo` would be detected. +Learn more about [Secret Detection](../secret_detection). ## Security Dashboard @@ -503,7 +470,12 @@ vulnerabilities in your groups, projects and pipelines. Read more about the Once a vulnerability is found, you can interact with it. Read more on how to [interact with the vulnerabilities](../index.md#interacting-with-the-vulnerabilities). -## Vulnerabilities database update +## Vulnerabilities database + +Vulnerabilities contained within the vulnerability database can be searched +and viewed at the [GitLab vulnerability advisory database](https://advisories.gitlab.com). + +### Vulnerabilities database update For more information about the vulnerabilities database update, check the [maintenance table](../index.md#maintenance-and-update-of-the-vulnerabilities-database). @@ -512,29 +484,29 @@ For more information about the vulnerabilities database update, check the For self-managed GitLab instances in an environment with limited, restricted, or intermittent access to external resources through the internet, some adjustments are required for the SAST job to -successfully run. For more information, see [Offline environments](../offline_deployments/index.md). +run successfully. For more information, see [Offline environments](../offline_deployments/index.md). ### Requirements for offline SAST To use SAST in an offline environment, you need: -- [Disable Docker-In-Docker](#disabling-docker-in-docker-for-sast) +- To keep Docker-In-Docker disabled (default). - GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). - Docker Container Registry with locally available copies of SAST [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), -meaning the runner will try to pull Docker images from the GitLab container registry even if a local +meaning the Runner tries to pull Docker images from the GitLab container registry even if a local copy is available. GitLab Runner's [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) in an offline environment if you prefer using only locally available Docker images. However, we -recommend keeping the pull policy setting to `always` as it will better enable updated scanners to -be utilized within your CI/CD pipelines. +recommend keeping the pull policy setting to `always` if not in an offline environment, as this +enables the use of updated scanners in your CI/CD pipelines. ### Make GitLab SAST analyzer images available inside your Docker registry For SAST with all [supported languages and frameworks](#supported-languages-and-frameworks), -import the following default SAST analyzer images from `registry.gitlab.com` to your local "offline" -registry: +import the following default SAST analyzer images from `registry.gitlab.com` into your +[local Docker container registry](../../packages/container_registry/index.md): ```plaintext registry.gitlab.com/gitlab-org/security-products/analyzers/bandit:2 @@ -557,7 +529,7 @@ registry.gitlab.com/gitlab-org/security-products/analyzers/tslint:2 The process for importing Docker images into a local offline Docker registry depends on **your network security policy**. Please consult your IT staff to find an accepted and approved process by which external resources can be imported or temporarily accessed. Note that these scanners are [updated periodically](../index.md#maintenance-and-update-of-the-vulnerabilities-database) -with new definitions, so consider if you are able to make periodic updates yourself. +with new definitions, so consider if you're able to make periodic updates yourself. For details on saving and transporting Docker images as a file, see Docker's documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), @@ -565,18 +537,15 @@ For details on saving and transporting Docker images as a file, see Docker's doc ### Set SAST CI job variables to use local SAST analyzers -[Override SAST environment variables](#customizing-the-sast-settings) to use to your [local container registry](./analyzers.md#using-a-custom-docker-mirror) -as the source for SAST analyzer images. - -For example, assuming a local Docker registry repository of `localhost:5000/analyzers`: +Add the following configuration to your `.gitlab-ci.yml` file. You must replace +`SAST_ANALYZER_IMAGE_PREFIX` to refer to your local Docker container registry: ```yaml include: - template: SAST.gitlab-ci.yml variables: - SAST_ANALYZER_IMAGE_PREFIX: "localhost:5000/analyzers" - SAST_DISABLE_DIND: "true" + SECURE_ANALYZERS_PREFIX: "localhost:5000/analyzers" ``` The SAST job should now use local copies of the SAST analyzers to scan your code and generate @@ -586,7 +555,7 @@ security reports without requiring internet access. ### Error response from daemon: error processing tar file: docker-tar: relocation error -This error occurs when the Docker version used to run the SAST job is `19.03.0`. -You are advised to update to Docker `19.03.1` or greater. Older versions are not +This error occurs when the Docker version that runs the SAST job is `19.03.0`. +Consider updating to Docker `19.03.1` or greater. Older versions are not affected. Read more in [this issue](https://gitlab.com/gitlab-org/gitlab/issues/13830#note_211354992 "Current SAST container fails"). diff --git a/doc/user/application_security/secret_detection/img/secret-detection-merge-request-ui.png b/doc/user/application_security/secret_detection/img/secret-detection-merge-request-ui.png Binary files differnew file mode 100644 index 00000000000..17893610f10 --- /dev/null +++ b/doc/user/application_security/secret_detection/img/secret-detection-merge-request-ui.png diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md new file mode 100644 index 00000000000..74dd3c89984 --- /dev/null +++ b/doc/user/application_security/secret_detection/index.md @@ -0,0 +1,67 @@ +--- +type: reference, howto +--- + +# Secret Detection **(ULTIMATE)** + +> [Introduced](https://about.gitlab.com/releases/2019/03/22/gitlab-11-9-released/#detect-secrets-and-credentials-in-the-repository) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.9. + +## Overview + +A recurring problem when developing applications is that developers may unintentionally commit +secrets and credentials to their remote repositories. If other people have access to the source, +or if the project is public, the sensitive information is then exposed and can be leveraged by +malicious users to gain access to resources like deployment environments. + +GitLab 11.9 includes a new check called Secret Detection. It scans the content of the repository +to find API keys and other information that should not be there. + +GitLab displays identified secrets as part of the SAST reports visibly in a few places: + +- [Security Dashboard](../security_dashboard/) +- Pipelines' **Security** tab +- Report in the merge request widget + + + +## Use cases + +- Detecting accidental commit of secrets like keys, passwords, and API tokens. +- Performing a single or recurring scan of the full history of your repository for secrets. + +## Configuration + +If you already have SAST enabled for your app, you don’t need to take any action to benefit from this +new feature. It is also included in the Auto DevOps default configuration. + +Secret Detection is performed by a [specific analyzer](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml#L180) +during the `sast` job. It runs regardless of the programming +language of your app, and you don't need to change your +CI/CD configuration file to enable it. Results are available in the SAST report. + +The Secret Detection analyzer includes [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) checks. + +NOTE: **Note:** +The Secret Detection analyzer will ignore "Password in URL" vulnerabilities if the password begins +with a dollar sign (`$`) as this likely indicates the password being used is an environment +variable. For example, `https://username:$password@example.com/path/to/repo` won't be +detected, whereas `https://username:password@example.com/path/to/repo` would be detected. + +## Full History Secret Scan + +GitLab 12.11 introduced support for scanning the full history of a reposity. This new functionality +is particularly useful when you are enabling Secret Detection in a repository for the first time and you +want to perform a full secret scan. Running a secret scan on the full history can take a long time, +especially for larger repositories with lengthy Git histories. We recommend not setting this variable +as part of your normal job defintion. + +A new configuration variable ([`SAST_GITLEAKS_HISTORIC_SCAN`](../sast/#vulnerability-filters)) +can be set to change the behavior of the GitLab Secret Detection scan to run on the entire Git history of a repository. + +We have created a [short video walkthrough](https://youtu.be/wDtc_K00Y0A) showcasing how you can perform a full history secret scan. +<div class="video-fallback"> + See the video: <a href="https://www.youtube.com/watch?v=wDtc_K00Y0A">Walkthrough of historical secret scan</a>. +</div> +<figure class="video-container"> + <iframe src="https://www.youtube.com/embed/wDtc_K00Y0A" frameborder="0" allowfullscreen="true"> </iframe> +</figure> diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_6.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_6.png Binary files differdeleted file mode 100644 index c93a3ce8c35..00000000000 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_6.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png Binary files differnew file mode 100644 index 00000000000..c788e2165ad --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png diff --git a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_export_csv_v13_0.png b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_export_csv_v13_0.png Binary files differnew file mode 100644 index 00000000000..77e75551bd9 --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_export_csv_v13_0.png diff --git a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v12_8.png b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v12_8.png Binary files differdeleted file mode 100644 index fd0548d0b34..00000000000 --- a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v12_8.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_0.png b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_0.png Binary files differnew file mode 100644 index 00000000000..a500f186c2b --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/instance_security_dashboard_with_projects_v13_0.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_export_csv_v12.10.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_export_csv_v12_10.png Binary files differindex 07b41b471d4..07b41b471d4 100644 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard_export_csv_v12.10.png +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_export_csv_v12_10.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png Binary files differdeleted file mode 100644 index 51e80bdb50d..00000000000 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png +++ /dev/null diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png Binary files differnew file mode 100644 index 00000000000..bb88b7f371c --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 42b28b7b9f2..2988b3642ef 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -30,7 +30,7 @@ To use the instance, group, project, or pipeline security dashboard: 1. At least one project inside a group must be configured with at least one of the [supported reports](#supported-reports). -1. The configured jobs must use the [new `reports` syntax](../../../ci/yaml/README.md#artifactsreports). +1. The configured jobs must use the [new `reports` syntax](../../../ci/pipelines/job_artifacts.md#artifactsreports). 1. [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or newer must be used. If you're using the shared Runners on GitLab.com, this is already the case. @@ -44,15 +44,17 @@ Visit the page for any pipeline which has run any of the [supported reports](#su  +NOTE: **Note:** +A pipeline consists of multiple jobs, including SAST and DAST scanning. If any job fails to finish for any reason, the security dashboard will not show SAST scanner output. For example, if the SAST job finishes but the DAST job fails, the security dashboard will not show SAST results. The analyzer will output an [exit code](../../../development/integrations/secure.md#exit-code) on failure. + ## Project Security Dashboard > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6165) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.1. -At the project level, the Security Dashboard displays the latest security reports -for your project from the last successful pipeline. Use it to find and fix vulnerabilities affecting the -[default branch](../../project/repository/branches/index.md#default-branch). +At the project level, the Security Dashboard displays the latest security reports for your project. +Use it to find and fix vulnerabilities. - + ### Export vulnerabilities @@ -64,7 +66,7 @@ NOTE: **Note:** It may take several minutes for the download to start if your project consists of thousands of vulnerabilities. Do not close the page until the download finishes. - + ## Group Security Dashboard @@ -78,32 +80,27 @@ First, navigate to the Security Dashboard found under your group's Once you're on the dashboard, at the top you should see a series of filters for: +- Status - Severity -- Confidence - Report type -- Project - -To the right of the filters, you should see a **Hide dismissed** toggle button. NOTE: **Note:** -The dashboard only shows projects with [security reports](#supported-reports) enabled in a group -according to the last successful projects' pipelines. +The dashboard only shows projects with [security reports](#supported-reports) enabled in a group. - + -Selecting one or more filters will filter the results in this page. Disabling the **Hide dismissed** -toggle button will let you also see vulnerabilities that have been dismissed. +Selecting one or more filters will filter the results in this page. The main section is a list of all the vulnerabilities in the group, sorted by severity. In that list, you can see the severity of the vulnerability, its name, its confidence (likelihood of the vulnerability to be a positive one), and the project it's from. -If you hover over a row, there will appear some actions you can take: +If you hover over a row, the following actions appear: -- "More info" -- "Create issue" -- "Dismiss vulnerability" +- More info +- Create issue +- Dismiss vulnerability Next to the list is a timeline chart that shows how many open vulnerabilities your projects had at various points in time. You can filter among 30, 60, and @@ -147,7 +144,23 @@ To add projects to the dashboard: Once added, the dashboard will display the vulnerabilities found in your chosen projects. - + + +### Export vulnerabilities + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213014) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. + +You can export all your vulnerabilities as CSV by clicking the **{upload}** **Export** +button located at top right of the **Instance Security Dashboard**. After the report +is built, the CSV report downloads to your local machine. The report contains all +vulnerabilities for the projects defined in the **Instance Security Dashboard**, +as filters don't apply to the export function. + +NOTE: **Note:** +It may take several minutes for the download to start if your project contains +thousands of vulnerabilities. Do not close the page until the download finishes. + + ## Keeping the dashboards up to date diff --git a/doc/user/application_security/threat_monitoring/index.md b/doc/user/application_security/threat_monitoring/index.md index 482fceea680..7bd148edd15 100644 --- a/doc/user/application_security/threat_monitoring/index.md +++ b/doc/user/application_security/threat_monitoring/index.md @@ -21,7 +21,7 @@ The Web Application Firewall section provides metrics for the NGINX Ingress controller and ModSecurity firewall. This section has the following prerequisites: -- Project has to have at least one [environment](../../../ci/environments.md). +- Project has to have at least one [environment](../../../ci/environments/index.md). - [Web Application Firewall](../../clusters/applications.md#web-application-firewall-modsecurity) has to be enabled. - [Elastic Stack](../../clusters/applications.md#web-application-firewall-modsecurity) has to be installed. @@ -38,7 +38,7 @@ about your Ingress traffic: If a significant percentage of traffic is anomalous, you should investigate it for potential threats by -[examining the application logs](../../clusters/applications.md#web-application-firewall-modsecurity). +[examining the Web Application Firewall logs](../../clusters/applications.md#web-application-firewall-modsecurity). ## Container Network Policy @@ -48,7 +48,7 @@ The **Container Network Policy** section provides packet flow metrics for your application's Kubernetes namespace. This section has the following prerequisites: -- Your project contains at least one [environment](../../../ci/environments.md) +- Your project contains at least one [environment](../../../ci/environments/index.md) - You've [installed Cilium](../../clusters/applications.md#install-cilium-using-gitlab-cicd) - You've configured the [Prometheus service](../../project/integrations/prometheus.md#enabling-prometheus-integration) diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index 5cb4f16e0d8..b691a97fc32 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -4,11 +4,7 @@ type: reference, howto # Standalone Vulnerability pages -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13561) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. - -CAUTION: **Warning:** -This feature is currently [Alpha](https://about.gitlab.com/handbook/product/#alpha-beta-ga). -You can begin using it, but it may receive important changes in the future. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13561) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. Each security vulnerability in the [Vulnerability List](../dependency_list/index.md) has its own standalone page. diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 47cbc0d4a1e..9ede9d9fdef 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -1,10 +1,16 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Managed Apps GitLab provides **GitLab Managed Apps**, a one-click install for various applications which can be added directly to your configured cluster. These applications are needed for [Review Apps](../../ci/review_apps/index.md) -and [deployments](../../ci/environments.md) when using [Auto DevOps](../../topics/autodevops/index.md). +and [deployments](../../ci/environments/index.md) when using [Auto DevOps](../../topics/autodevops/index.md). You can install them after you [create a cluster](../project/clusters/add_remove_clusters.md). @@ -128,9 +134,9 @@ before deploying one. NOTE: **Note:** The [`runner/gitlab-runner`](https://gitlab.com/gitlab-org/charts/gitlab-runner) -chart is used to install this application with a -[`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/runner/values.yaml) -file. Customizing installation by modifying this file is not supported. +chart is used to install this application, using +[a preconfigured `values.yaml`](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/blob/master/values.yaml) +file. Customizing the installation by modifying this file is not supported. ### Ingress @@ -314,6 +320,16 @@ To change your WAF's mode: 1. Under **Global default**, select your desired mode. 1. Click **Save changes**. +##### WAF version updates + +Enabling, disabling, or changing the logging mode for **ModSecurity** is only allowed within same version of [Ingress](#ingress) due to limitations in [Helm](https://helm.sh/) which might be overcome in future releases. + +**ModSecurity** UI controls are disabled if the version deployed differs from the one available in GitLab, while actions at the [Ingress](#ingress) level, such as uninstalling, can still be performed: + + + +Updating [Ingress](#ingress) to the most recent version enables you to take advantage of bug fixes, security fixes, and performance improvements. To update [Ingress application](#ingress), you must first uninstall it, and then re-install it as described in [Install ModSecurity](../../topics/web_application_firewall/quick_start_guide.md). + ##### Viewing Web Application Firewall traffic > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14707) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. @@ -356,7 +372,7 @@ will also see ready-to-use DevOps Runbooks built with Nurtch's [Rubix library](h More information on creating executable runbooks can be found in [our Runbooks -documentation](../project/clusters/runbooks/index.md#executable-runbooks). Note that +documentation](../project/clusters/runbooks/index.md#configure-an-executable-runbook-with-gitlab). Note that Ingress must be installed and have an IP address assigned before JupyterHub can be installed. @@ -487,18 +503,25 @@ and you will have access to more advanced querying capabilities. Log data is automatically deleted after 30 days using [Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/5.5/about.html). -To enable log shipping, install Elastic Stack into the cluster with the **Install** button. +To enable log shipping: + +1. Ensure your cluster contains at least 3 nodes of instance types larger than + `f1-micro`, `g1-small`, or `n1-standard-1`. +1. Navigate to **{cloud-gear}** **Operations > Kubernetes**. +1. In **Kubernetes Cluster**, select a cluster. +1. In the **Applications** section, find **Elastic Stack** and click **Install**. NOTE: **Note:** -The [`stable/elastic-stack`](https://github.com/helm/charts/tree/master/stable/elastic-stack) +The [`gitlab/elastic-stack`](https://gitlab.com/gitlab-org/charts/elastic-stack) chart is used to install this application with a [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/elastic_stack/values.yaml) file. NOTE: **Note:** -The chart will deploy 5 Elasticsearch nodes: 2 masters, 2 data and 1 client node, -with resource requests totalling 0.125 CPU and 4.5GB RAM. Each data node requests 1.5GB of memory, -which makes it incompatible with clusters of `f1-micro` and `g1-small` instance types. +The chart deploys 3 identical Elasticsearch pods which can't be colocated, and each +require 1 CPU and 2 GB of RAM, making them incompatible with clusters containing +fewer than 3 nodes or consisting of `f1-micro`, `g1-small`, `n1-standard-1`, or +`*-highcpu-2` instance types. NOTE: **Note:** The Elastic Stack cluster application is intended as a log aggregation solution and is not related to our @@ -517,25 +540,25 @@ Save the following to `kibana.yml`: elasticsearch: enabled: false -logstash: +filebeat: enabled: false kibana: enabled: true - env: - ELASTICSEARCH_HOSTS: http://elastic-stack-elasticsearch-client.gitlab-managed-apps.svc.cluster.local:9200 + elasticsearchHosts: http://elastic-stack-elasticsearch-master.gitlab-managed-apps.svc.cluster.local:9200 ``` Then install it on your cluster: ```shell -helm install --name kibana stable/elastic-stack --values kibana.yml +helm repo add gitlab https://charts.gitlab.io +helm install --name kibana gitlab/elastic-stack --values kibana.yml ``` -To access kibana, forward the port to your local machine: +To access Kibana, forward the port to your local machine: ```shell -kubectl port-forward svc/kibana 5601:443 +kubectl port-forward svc/kibana-kibana 5601:5601 ``` Then, you can visit Kibana at `http://localhost:5601`. @@ -556,11 +579,10 @@ To enable Fluentd: 1. Provide the host domain name or URL in **SIEM Hostname**. 1. Provide the host port number in **SIEM Port**. 1. Select a **SIEM Protocol**. -1. Check **Send ModSecurity Logs**. If you do not select this checkbox, the **Install** - button is disabled. +1. Select at least one of the available logs (such as WAF or Cilium). 1. Click **Save changes**. - + ### Future apps @@ -777,7 +799,7 @@ In order for GitLab Runner to function, you **must** specify the following: - `runnerRegistrationToken` - The registration token for adding new Runners to GitLab. This must be [retrieved from your GitLab instance](../../ci/runners/README.md). -These values can be specifed using [CI variables](../../ci/variables/README.md): +These values can be specified using [CI variables](../../ci/variables/README.md): - `GITLAB_RUNNER_GITLAB_URL` will be used for `gitlabUrl`. - `GITLAB_RUNNER_REGISTRATION_TOKEN` will be used for `runnerRegistrationToken` @@ -792,10 +814,12 @@ available configuration options. > [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/22) in GitLab 12.8. -[Cilium](https://cilium.io/) is a networking plugin for Kubernetes -that you can use to implement support for -[NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -resources. For more information on [Network Policies](../../topics/autodevops/stages.md#network-policy), see the documentation. +[Cilium](https://cilium.io/) is a networking plugin for Kubernetes that you can use to implement +support for [NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +resources. For more information, see [Network Policies](../../topics/autodevops/stages.md#network-policy). + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see the [Container Network Security Demo for GitLab 12.8](https://www.youtube.com/watch?v=pgUEdhdhoUI). Enable Cilium in the `.gitlab/managed-apps/config.yaml` file to install it: @@ -822,7 +846,8 @@ management project. Refer to the for the available configuration options. CAUTION: **Caution:** -Installation and removal of the Cilium [requires restart](https://cilium.readthedocs.io/en/stable/gettingstarted/k8s-install-gke/#restart-remaining-pods) +Installation and removal of the Cilium requires a **manual** +[restart](https://cilium.readthedocs.io/en/stable/gettingstarted/k8s-install-gke/#restart-remaining-pods) of all affected pods in all namespaces to ensure that they are [managed](https://cilium.readthedocs.io/en/stable/troubleshooting/#ensure-pod-is-managed-by-cilium) by the correct networking plugin. @@ -908,15 +933,15 @@ vault: installed: true ``` -By default you will get a basic Vault setup with no high availability nor any scalable -storage backend. This is enough for simple testing and small scale deployments, though has limits +By default you will get a basic Vault setup with no scalable +storage backend. This is enough for simple testing and small-scale deployments, though has limits to how much it can scale, and as it is a single instance deployment, you will experience downtime when upgrading the Vault application. To optimally use Vault in a production environment, it's ideal to have a good understanding of the internals of Vault and how to configure it. This can be done by reading the [the Vault documentation](https://www.vaultproject.io/docs/internals/) as well as -the Vault Helm chart [values.yaml file](https://github.com/hashicorp/vault-helm/blob/v0.3.3/values.yaml). +the Vault Helm chart [`values.yaml` file](https://github.com/hashicorp/vault-helm/blob/v0.3.3/values.yaml). At a minimum you will likely set up: @@ -1009,11 +1034,11 @@ In addition, the following variables must be specified using [CI variables](../. | CI Variable | Description | |:---------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `JUPYTERHUB_PROXY_SECRET_TOKEN` | Sets [`proxy.secretToken`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference.html#proxy-secrettoken). Generate using `openssl rand -hex 32`. | -| `JUPYTERHUB_COOKIE_SECRET` | Sets [`hub.cookieSecret`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference.html#hub-cookiesecret). Generate using `openssl rand -hex 32`. | +| `JUPYTERHUB_PROXY_SECRET_TOKEN` | Secure string used for signing communications from the hub. See[`proxy.secretToken`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#proxy-secrettoken). | +| `JUPYTERHUB_COOKIE_SECRET` | Secure string used for signing secure cookies. See [`hub.cookieSecret`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#hub-cookiesecret). | | `JUPYTERHUB_HOST` | Hostname used for the installation. For example, `jupyter.gitlab.example.com`. | | `JUPYTERHUB_GITLAB_HOST` | Hostname of the GitLab instance used for authentication. For example, `gitlab.example.com`. | -| `JUPYTERHUB_AUTH_CRYPTO_KEY` | Sets [`auth.state.cryptoKey`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference.html#auth-state-cryptokey). Generate using `openssl rand -hex 32`. | +| `JUPYTERHUB_AUTH_CRYPTO_KEY` | A 32-byte encryption key used to set [`auth.state.cryptoKey`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#auth-state-cryptokey). | | `JUPYTERHUB_AUTH_GITLAB_CLIENT_ID` | "Application ID" for the OAuth Application. | | `JUPYTERHUB_AUTH_GITLAB_CLIENT_SECRET` | "Secret" for the OAuth Application. | @@ -1042,12 +1067,12 @@ elasticStack: Elastic Stack is installed into the `gitlab-managed-apps` namespace of your cluster. -You can check the default [values.yaml](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/elastic_stack/values.yaml) we set for this chart. +You can check the default [`values.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/elastic_stack/values.yaml) we set for this chart. You can customize the installation of Elastic Stack by defining `.gitlab/managed-apps/elastic-stack/values.yaml` file in your cluster management project. Refer to the -[chart](https://github.com/helm/charts/blob/master/stable/elastic-stack/values.yaml) for the +[chart](https://gitlab.com/gitlab-org/charts/elastic-stack) for the available configuration options. NOTE: **Note:** @@ -1070,7 +1095,7 @@ Crossplane: Crossplane is installed into the `gitlab-managed-apps` namespace of your cluster. You can check the default -[values.yaml](https://github.com/crossplane/crossplane/blob/master/cluster/charts/crossplane/values.yaml.tmpl) +[`values.yaml`](https://github.com/crossplane/crossplane/blob/master/cluster/charts/crossplane/values.yaml.tmpl) we set for this chart. You can customize the installation of Crossplane by defining @@ -1090,7 +1115,7 @@ Fluentd: installed: true ``` -You can also review the default values set for this chart in the [values.yaml](https://github.com/helm/charts/blob/master/stable/fluentd/values.yaml) file. +You can also review the default values set for this chart in the [`values.yaml`](https://github.com/helm/charts/blob/master/stable/fluentd/values.yaml) file. You can customize the installation of Fluentd by defining `.gitlab/managed-apps/fluentd/values.yaml` file in your cluster management @@ -1207,7 +1232,7 @@ epic](https://gitlab.com/groups/gitlab-org/-/epics/1201). Applications can fail with the following error: -```text +```plaintext Error: remote error: tls: bad certificate ``` diff --git a/doc/user/clusters/crossplane.md b/doc/user/clusters/crossplane.md index 4e2ae87ecb9..a9a5f768ec8 100644 --- a/doc/user/clusters/crossplane.md +++ b/doc/user/clusters/crossplane.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Crossplane configuration Once Crossplane [is installed](applications.md#crossplane), it must be configured for @@ -161,7 +167,7 @@ metadata: specTemplate: writeConnectionSecretsToNamespace: gitlab-managed-apps forProvider: - databaseVersion: POSTGRES_9_6 + databaseVersion: POSTGRES_11_7 region: $REGION settings: tier: db-custom-1-3840 @@ -183,7 +189,7 @@ metadata: specTemplate: writeConnectionSecretsToNamespace: gitlab-managed-apps forProvider: - databaseVersion: POSTGRES_9_6 + databaseVersion: POSTGRES_11_7 region: $REGION settings: tier: db-custom-1-3840 diff --git a/doc/user/clusters/environments.md b/doc/user/clusters/environments.md index f83be85726a..a2adf238dda 100644 --- a/doc/user/clusters/environments.md +++ b/doc/user/clusters/environments.md @@ -1,9 +1,15 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Cluster Environments **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13392) for group-level clusters in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14809) for instance-level clusters in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. -Cluster environments provide a consolidated view of which CI [environments](../../ci/environments.md) are +Cluster environments provide a consolidated view of which CI [environments](../../ci/environments/index.md) are deployed to the Kubernetes cluster and it: - Shows the project and the relevant environment related to the deployment. diff --git a/doc/user/clusters/img/fluentd_v12_10.png b/doc/user/clusters/img/fluentd_v12_10.png Binary files differdeleted file mode 100644 index e8c5c832020..00000000000 --- a/doc/user/clusters/img/fluentd_v12_10.png +++ /dev/null diff --git a/doc/user/clusters/img/fluentd_v13_0.png b/doc/user/clusters/img/fluentd_v13_0.png Binary files differnew file mode 100644 index 00000000000..edc73285238 --- /dev/null +++ b/doc/user/clusters/img/fluentd_v13_0.png diff --git a/doc/user/clusters/management_project.md b/doc/user/clusters/management_project.md index 2b8ed83bdb2..03b4dc45015 100644 --- a/doc/user/clusters/management_project.md +++ b/doc/user/clusters/management_project.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Cluster management project (alpha) CAUTION: **Warning:** diff --git a/doc/user/compliance/license_compliance/img/license_compliance.png b/doc/user/compliance/license_compliance/img/license_compliance.png Binary files differdeleted file mode 100644 index cdce6b5fe38..00000000000 --- a/doc/user/compliance/license_compliance/img/license_compliance.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/license_compliance_add_license_v12_3.png b/doc/user/compliance/license_compliance/img/license_compliance_add_license_v12_3.png Binary files differdeleted file mode 100644 index ea4db16284c..00000000000 --- a/doc/user/compliance/license_compliance/img/license_compliance_add_license_v12_3.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/license_compliance_add_license_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_add_license_v13_0.png Binary files differnew file mode 100644 index 00000000000..992c08edcd3 --- /dev/null +++ b/doc/user/compliance/license_compliance/img/license_compliance_add_license_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_decision.png b/doc/user/compliance/license_compliance/img/license_compliance_decision.png Binary files differdeleted file mode 100644 index fbf90bec7fd..00000000000 --- a/doc/user/compliance/license_compliance/img/license_compliance_decision.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/license_compliance_decision_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_decision_v13_0.png Binary files differnew file mode 100644 index 00000000000..d6c6142c0e7 --- /dev/null +++ b/doc/user/compliance/license_compliance/img/license_compliance_decision_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v12_3.png b/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v12_3.png Binary files differdeleted file mode 100644 index fd519d63b3e..00000000000 --- a/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v12_3.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v13_0.png Binary files differnew file mode 100644 index 00000000000..9ae59e2b96b --- /dev/null +++ b/doc/user/compliance/license_compliance/img/license_compliance_pipeline_tab_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_search_v12_3.png b/doc/user/compliance/license_compliance/img/license_compliance_search_v12_3.png Binary files differdeleted file mode 100644 index 4a7cff2e85c..00000000000 --- a/doc/user/compliance/license_compliance/img/license_compliance_search_v12_3.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/license_compliance_search_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_search_v13_0.png Binary files differnew file mode 100644 index 00000000000..8ee55003768 --- /dev/null +++ b/doc/user/compliance/license_compliance/img/license_compliance_search_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_settings_v12_3.png b/doc/user/compliance/license_compliance/img/license_compliance_settings_v12_3.png Binary files differdeleted file mode 100644 index 72d0888a9dc..00000000000 --- a/doc/user/compliance/license_compliance/img/license_compliance_settings_v12_3.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/license_compliance_settings_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_settings_v13_0.png Binary files differnew file mode 100644 index 00000000000..52b26abd9c5 --- /dev/null +++ b/doc/user/compliance/license_compliance/img/license_compliance_settings_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_compliance_v13_0.png b/doc/user/compliance/license_compliance/img/license_compliance_v13_0.png Binary files differnew file mode 100644 index 00000000000..dc227bf05ef --- /dev/null +++ b/doc/user/compliance/license_compliance/img/license_compliance_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/license_list_v12_6.png b/doc/user/compliance/license_compliance/img/license_list_v12_6.png Binary files differdeleted file mode 100644 index 8f2b510be0d..00000000000 --- a/doc/user/compliance/license_compliance/img/license_list_v12_6.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/license_list_v13_0.png b/doc/user/compliance/license_compliance/img/license_list_v13_0.png Binary files differnew file mode 100644 index 00000000000..3964c837c6a --- /dev/null +++ b/doc/user/compliance/license_compliance/img/license_list_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_add_v12_9.png b/doc/user/compliance/license_compliance/img/policies_maintainer_add_v12_9.png Binary files differdeleted file mode 100644 index ad5a49eebe5..00000000000 --- a/doc/user/compliance/license_compliance/img/policies_maintainer_add_v12_9.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_0.png b/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_0.png Binary files differnew file mode 100644 index 00000000000..8070e2cb1a5 --- /dev/null +++ b/doc/user/compliance/license_compliance/img/policies_maintainer_add_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v12_9.png b/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v12_9.png Binary files differdeleted file mode 100644 index 4f2380a0bf6..00000000000 --- a/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v12_9.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_0.png b/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_0.png Binary files differnew file mode 100644 index 00000000000..741d1237751 --- /dev/null +++ b/doc/user/compliance/license_compliance/img/policies_maintainer_edit_v13_0.png diff --git a/doc/user/compliance/license_compliance/img/policies_v12_9.png b/doc/user/compliance/license_compliance/img/policies_v12_9.png Binary files differdeleted file mode 100644 index b3bca716ae5..00000000000 --- a/doc/user/compliance/license_compliance/img/policies_v12_9.png +++ /dev/null diff --git a/doc/user/compliance/license_compliance/img/policies_v13_0.png b/doc/user/compliance/license_compliance/img/policies_v13_0.png Binary files differnew file mode 100644 index 00000000000..4712d2b7aba --- /dev/null +++ b/doc/user/compliance/license_compliance/img/policies_v13_0.png diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 2e771a17163..cbabed00283 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -30,12 +30,20 @@ will be displayed in the merge request area. That is the case when you add the Consecutive merge requests will have something to compare to and the license compliance report will be shown properly. - + If you are a project or group Maintainer, you can click on a license to be given the choice to allow it or deny it. - + + +When GitLab detects a **Denied** license, you can view it in the [license list](#license-list). + + + +You can view and modify existing policies from the [policies](#policies) tab. + + ## Use cases @@ -49,19 +57,30 @@ The following languages and package managers are supported. | Language | Package managers | Scan Tool | |------------|-------------------------------------------------------------------|----------------------------------------------------------| -| JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/), [yarn](https://yarnpkg.com/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Go | [Godep](https://github.com/tools/godep), go get ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)), gvt ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)), glide ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)), dep ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)), trash ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)) and govendor ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)), [go mod](https://github.com/golang/go/wiki/Modules) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Go | [Godep](https://github.com/tools/godep), [go mod](https://github.com/golang/go/wiki/Modules) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Java | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| .NET | [Nuget](https://www.nuget.org/) (.NET Framework is supported via the [mono project](https://www.mono-project.com/). Windows specific dependencies are not supported at this time.) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| .NET | [Nuget](https://www.nuget.org/) (.NET Framework is supported via the [mono project](https://www.mono-project.com/). Windows specific dependencies are not supported at this time.) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Python | [pip](https://pip.pypa.io/en/stable/) (Python is supported through [requirements.txt](https://pip.readthedocs.io/en/1.1/requirements.html) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock).) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Ruby | [gem](https://rubygems.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Erlang | [rebar](https://www.rebar3.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| -| Objective-C, Swift | [Carthage](https://github.com/Carthage/Carthage) , [CocoaPods v0.39 and below](https://cocoapods.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Elixir | [mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| C++/C | [conan](https://conan.io/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| -| Scala | [sbt](https://www.scala-sbt.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| -| Rust | [cargo](https://crates.io/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| -| PHP | [composer](https://getcomposer.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)| +| Ruby | [gem](https://rubygems.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Objective-C, Swift | [Carthage](https://github.com/Carthage/Carthage) |[License Finder](https://github.com/pivotal/LicenseFinder)| + +### Experimental support + +The following languages and package managers are [supported experimentally](https://github.com/pivotal/LicenseFinder#experimental-project-types), +which means that the reported licenses might be incomplete or inaccurate. + +| Language | Package managers | Scan Tool | +|------------|-------------------------------------------------------------------|----------------------------------------------------------| +| JavaScript | [yarn](https://yarnpkg.com/)|[License Finder](https://github.com/pivotal/LicenseFinder)| +| Go | go get, gvt, glide, dep, trash, govendor |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Erlang | [rebar](https://www.rebar3.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Objective-C, Swift | [CocoaPods](https://cocoapods.org/) v0.39 and below |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Elixir | [mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| C++/C | [conan](https://conan.io/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Scala | [sbt](https://www.scala-sbt.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Rust | [cargo](https://crates.io/) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| PHP | [composer](https://getcomposer.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| ## Requirements @@ -81,7 +100,7 @@ For GitLab versions earlier than 11.9, you can copy and use the job as defined that template. NOTE: **Note:** -In GitLab 13.0, the `License-Management.gitlab-ci.yml` template is scheduled to be removed. +GitLab 13.0 removes the `License-Management.gitlab-ci.yml` template. Use `License-Scanning.gitlab-ci.yml` instead. Add the following to your `.gitlab-ci.yml` file: @@ -96,12 +115,12 @@ and scan your dependencies to find their licenses. NOTE: **Note:** Before GitLab 12.8, the `license_scanning` job was named `license_management`. -In GitLab 13.0, the `license_management` job is scheduled to be removed completely, +GitLab 13.0 removes the `license_management` job, so you're advised to migrate to the `license_scanning` job and used the new `License-Scanning.gitlab-ci.yml` template. The results will be saved as a -[License Compliance report artifact](../../../ci/yaml/README.md#artifactsreportslicense_scanning-ultimate) +[License Compliance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportslicense_scanning-ultimate) that you can later download and analyze. Due to implementation limitations, we always take the latest License Compliance artifact available. Behind the scenes, the [GitLab License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) @@ -114,15 +133,17 @@ The License Compliance settings can be changed through [environment variables](# License Compliance can be configured using environment variables. -| Environment variable | Required | Description | -|-----------------------|----------|-------------| -| `MAVEN_CLI_OPTS` | no | Additional arguments for the mvn executable. If not supplied, defaults to `-DskipTests`. | -| `LICENSE_FINDER_CLI_OPTS` | no | Additional arguments for the `license_finder` executable. For example, if your project has both Golang and Ruby code stored in different directories and you want to only scan the Ruby code, you can update your `.gitlab-ci-yml` template to specify which project directories to scan, like `LICENSE_FINDER_CLI_OPTS: '--debug --aggregate-paths=. ruby'`. | -| `LM_JAVA_VERSION` | no | Version of Java. If set to `11`, Maven and Gradle use Java 11 instead of Java 8. | -| `LM_PYTHON_VERSION` | no | Version of Python. If set to `3`, dependencies are installed using Python 3 instead of Python 2.7. | -| `SETUP_CMD` | no | Custom setup for the dependency installation. (experimental) | -| `PIP_INDEX_URL` | no | Base URL of Python Package Index (default: `https://pypi.org/simple/`). | -| `ADDITIONAL_CA_CERT_BUNDLE` | no | Bundle of trusted CA certificates (currently supported in Python projects). | +| Environment variable | Required | Description | +|-----------------------------|----------|-------------| +| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address to download the analyzer from. | +| `ADDITIONAL_CA_CERT_BUNDLE` | no | Bundle of trusted CA certificates (currently supported in Pip, Pipenv, Maven, Gradle, Yarn, and NPM projects). | +| `GRADLE_CLI_OPTS` | no | Additional arguments for the gradle executable. If not supplied, defaults to `--exclude-task=test`. | +| `LICENSE_FINDER_CLI_OPTS` | no | Additional arguments for the `license_finder` executable. For example, if your project has both Golang and Ruby code stored in different directories and you want to only scan the Ruby code, you can update your `.gitlab-ci-yml` template to specify which project directories to scan, like `LICENSE_FINDER_CLI_OPTS: '--debug --aggregate-paths=. ruby'`. | +| `LM_JAVA_VERSION` | no | Version of Java. If set to `11`, Maven and Gradle use Java 11 instead of Java 8. | +| `LM_PYTHON_VERSION` | no | Version of Python. If set to `3`, dependencies are installed using Python 3 instead of Python 2.7. | +| `MAVEN_CLI_OPTS` | no | Additional arguments for the mvn executable. If not supplied, defaults to `-DskipTests`. | +| `PIP_INDEX_URL` | no | Base URL of Python Package Index (default: `https://pypi.org/simple/`). | +| `SETUP_CMD` | no | Custom setup for the dependency installation (experimental). | ### Installing custom dependencies @@ -263,21 +284,80 @@ license_scanning: The [`pip.conf`](https://pip.pypa.io/en/stable/reference/pip/) allows you to specify a list of [trusted hosts](https://pip.pypa.io/en/stable/reference/pip/#cmdoption-trusted-host): -```text +```plaintext [global] trusted-host = pypi.example.com ``` +#### Using private Python repos + +If you have a private Python repository you can use the `PIP_INDEX_URL` [environment variable](#available-variables) +to specify its location. It's also possible to provide a custom `pip.conf` for +[additional configuration](#custom-root-certificates-for-python). + +### Configuring NPM projects + +You can configure NPM projects by using an [`.npmrc`](https://docs.npmjs.com/configuring-npm/npmrc.html) +file. + +#### Using private NPM registries + +If you have a private NPM registry you can use the +[`registry`](https://docs.npmjs.com/using-npm/config#registry) +setting to specify its location. + +For example: + +```plaintext +registry = https://npm.example.com +``` + +#### Custom root certificates for NPM + +You can supply a custom root certificate to complete TLS verification by using the +`ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables). + +To disable TLS verification you can provide the [`strict-ssl`](https://docs.npmjs.com/using-npm/config#strict-ssl) +setting. + +For example: + +```plaintext +strict-ssl = false +``` + +### Configuring Yarn projects + +You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc) +file. + +#### Using private Yarn registries + +If you have a private Yarn registry you can use the +[`npmRegistryServer`](https://yarnpkg.com/configuration/yarnrc#npmRegistryServer) +setting to specify its location. + +For example: + +```text +npmRegistryServer: "https://npm.example.com" +``` + +#### Custom root certificates for Yarn + +You can supply a custom root certificate to complete TLS verification by using the +`ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables). + ### Migration from `license_management` to `license_scanning` In GitLab 12.8 a new name for `license_management` job was introduced. This change was made to improve clarity around the purpose of the scan, which is to scan and collect the types of licenses present in a projects dependencies. -The support of `license_management` is scheduled to be dropped in GitLab 13.0. +GitLab 13.0 drops support for `license_management`. If you're using a custom setup for License Compliance, you're required to update your CI config accordingly: 1. Change the CI template to `License-Scanning.gitlab-ci.yml`. 1. Change the job name to `license_scanning` (if you mention it in `.gitlab-ci.yml`). -1. Change the artifact name to `gl-license-scanning-report.json` (if you mention it in `.gitlab-ci.yml`). +1. Change the artifact name to `license_scanning`, and the file name to `gl-license-scanning-report.json` (if you mention it in `.gitlab-ci.yml`). For example, the following `.gitlab-ci.yml`: @@ -303,11 +383,21 @@ license_scanning: license_scanning: gl-license-scanning-report.json ``` +If you use the `license_management` artifact in GitLab 13.0 or later, the License Compliance job generates this error: + +```plaintext +WARNING: Uploading artifacts to coordinator... failed id=:id responseStatus=400 Bad Request status=400 Bad Request token=:sha + +FATAL: invalid_argument +``` + +If you encounter this error, follow the instructions described in this section. + ## Running License Compliance in an offline environment For self-managed GitLab instances in an environment with limited, restricted, or intermittent access to external resources through the internet, some adjustments are required for the License Compliance job to -successfully run. +successfully run. For more information, see [Offline environments](../../application_security/offline_deployments/index.md). ### Requirements for offline License Compliance @@ -318,11 +408,11 @@ To use License Compliance in an offline environment, you need: NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), -meaning the runner will try to pull Docker images from the GitLab container registry even if a local +meaning the Runner tries to pull Docker images from the GitLab container registry even if a local copy is available. GitLab Runner's [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) in an offline environment if you prefer using only locally available Docker images. However, we -recommend leaving the pull policy set to `always`, as it better enables updated scanners to be used -within your CI/CD pipelines. +recommend keeping the pull policy setting to `always` if not in an offline environment, as this +enables the use of updated scanners in your CI/CD pipelines. ### Make GitLab License Compliance analyzer images available inside your Docker registry @@ -345,10 +435,8 @@ For details on saving and transporting Docker images as a file, see Docker's doc ### Set License Compliance CI job variables to use local License Compliance analyzers -Override License Compliance environment variables to use to your local container registry -as the source for License Compliance analyzer images. - -For example, this assumes a local Docker registry repository of `localhost:5000/analyzers`: +Add the following configuration to your `.gitlab-ci.yml` file. You must replace `image` to refer to +the License Compliance Docker image hosted on your local Docker container registry: ```yaml include: @@ -362,8 +450,8 @@ license_scanning: The License Compliance job should now use local copies of the License Compliance analyzers to scan your code and generate security reports, without requiring internet access. -Additional [configuration](#using-private-maven-repos) may be needed for connecting to private Maven -repositories. +Additional configuration may be needed for connecting to [private Maven repositories](#using-private-maven-repos), +[private NPM registries](#using-private-npm-registries), [private Yarn registries](#using-private-yarn-registries), and [private Python repositories](#using-private-python-repos). Exact name matches are required for [project policies](#project-policies-for-license-compliance) when running in an offline environment ([see related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212388)). @@ -384,7 +472,7 @@ To allow or deny a license: **License Compliance** section. 1. Click the **Add a license** button. -  +  1. In the **License name** dropdown, either: - Select one of the available licenses. You can search for licenses in the field @@ -398,23 +486,23 @@ To modify an existing license: 1. In the **License Compliance** list, click the **Allow/Deny** dropdown to change it to the desired status. -  +  Searching for Licenses: 1. Use the **Search** box to search for a specific license. -  +  ## License Compliance report under pipelines > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5491) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2. From your project's left sidebar, navigate to **CI/CD > Pipelines** and click on the -pipeline ID that has a `license_management` job to see the Licenses tab with the listed +pipeline ID that has a `license_scanning` job to see the Licenses tab with the listed licenses (if any). - + <!-- ## Troubleshooting @@ -447,8 +535,9 @@ in your project's sidebar, and you'll see the licenses displayed, where: - **Name:** The name of the license. - **Component:** The components which have this license. +- **Policy Violation:** The license has a [license policy](#policies) marked as **Deny**. - + ## Policies @@ -459,9 +548,9 @@ and the associated classifications for each. Policies can be configured by maintainers of the project. - - + + Developers of the project can view the policies configured in a project. - + diff --git a/doc/user/feature_highlight.md b/doc/user/feature_highlight.md index df85a129ce1..47f8671afae 100644 --- a/doc/user/feature_highlight.md +++ b/doc/user/feature_highlight.md @@ -1,6 +1,6 @@ # Feature highlight -> [Introduced][ce-16379] in GitLab 10.5 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/16379) in GitLab 10.5 Feature highlights are represented by a pulsing blue dot. Hovering over the dot will open up callout with more information. @@ -11,5 +11,3 @@ at the bottom of the callout. There isn't a way to restore the feature highlight after it has been dismissed.  - -[ce-16379]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/16379 diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 7c005a60bea..d615b67f3d2 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -28,12 +28,12 @@ gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAA ## Mail configuration -GitLab.com sends emails from the `mg.gitlab.com` domain via [Mailgun] and has +GitLab.com sends emails from the `mg.gitlab.com` domain via [Mailgun](https://www.mailgun.com/) and has its own dedicated IP address (`198.61.254.240`). ## Alternative SSH port -GitLab.com can be reached via a [different SSH port][altssh] for `git+ssh`. +GitLab.com can be reached via a [different SSH port](https://about.gitlab.com/blog/2016/02/18/gitlab-dot-com-now-supports-an-alternate-git-plus-ssh-port/) for `git+ssh`. | Setting | Value | | --------- | ------------------- | @@ -89,7 +89,7 @@ or over the size limit, you can [reduce your repository size with Git](../projec | Repository size including LFS | 10G | Unlimited | NOTE: **Note:** -A single `git push` is limited to 5GB. LFS is not affected by this limit. +`git push` and GitLab project imports are limited to 5GB per request. Git LFS and imports other than a file upload are not affected by this limit. ## IP range @@ -115,9 +115,12 @@ A limit of: GitLab offers Linux and Windows shared runners hosted on GitLab.com for executing your pipelines. +NOTE: **Note:** +Shared Runners provided by GitLab are **not** configurable. Consider [installing your own Runner](https://docs.gitlab.com/runner/install/) if you have specific configuration needs. + ### Linux Shared Runners -Linux Shared Runners on GitLab.com run in [autoscale mode] and are powered by Google Cloud Platform. +Linux Shared Runners on GitLab.com run in [autoscale mode](https://docs.gitlab.com/runner/configuration/autoscale.html) and are powered by Google Cloud Platform. Autoscaling means reduced waiting times to spin up CI/CD jobs, and isolated VMs for each project, thus maximizing security. They're free to use for public open source projects and limited to 2000 CI minutes per month per group for private projects. More minutes @@ -133,16 +136,36 @@ The `gitlab-shared-runners-manager-X.gitlab.com` fleet of Runners are dedicated Jobs handled by the shared Runners on GitLab.com (`shared-runners-manager-X.gitlab.com`), **will be timed out after 3 hours**, regardless of the timeout configured in a -project. Check the issues [4010] and [4070] for the reference. +project. Check the issues [4010](https://gitlab.com/gitlab-com/infrastructure/issues/4010) and [4070](https://gitlab.com/gitlab-com/infrastructure/issues/4070) for the reference. Below are the shared Runners settings. | Setting | GitLab.com | Default | | ----------- | ----------------- | ---------- | -| [GitLab Runner] | [Runner versions dashboard](https://dashboards.gitlab.com/d/000000159/ci?from=now-1h&to=now&refresh=5m&orgId=1&panelId=12&fullscreen&theme=light) | - | +| [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) | [Runner versions dashboard](https://dashboards.gitlab.com/d/000000159/ci?from=now-1h&to=now&refresh=5m&orgId=1&panelId=12&fullscreen&theme=light) | - | | Executor | `docker+machine` | - | | Default Docker image | `ruby:2.5` | - | -| `privileged` (run [Docker in Docker]) | `true` | `false` | +| `privileged` (run [Docker in Docker](https://hub.docker.com/_/docker/)) | `true` | `false` | + +#### Pre-clone script + +Linux Shared Runners on GitLab.com provide a way to run commands in a CI +job before the Runner attempts to run `git init` and `git fetch` to +download a GitLab repository. The +[pre_clone_script](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) +can be used for: + +- Seeding the build directory with repository data +- Sending a request to a server +- Downloading assets from a CDN +- Any other commands that must run before the `git init` + +To use this feature, define a [CI/CD variable](../../ci/variables/README.md#create-a-custom-variable-in-the-ui) called +`CI_PRE_CLONE_SCRIPT` that contains a bash script. + +[This example](../../development/pipelines.md#pre-clone-step) +demonstrates how you might use a pre-clone step to seed the build +directory. #### `config.toml` @@ -164,6 +187,7 @@ sentry_dsn = "X" request_concurrency = X url = "https://gitlab.com/" token = "SHARED_RUNNER_TOKEN" + pre_clone_script = "eval \"$CI_PRE_CLONE_SCRIPT\"" executor = "docker+machine" environment = [ "DOCKER_DRIVER=overlay2", @@ -432,7 +456,7 @@ of proposed changes can be found at ## Unicorn -GitLab.com adjusts the memory limits for the [unicorn-worker-killer][unicorn-worker-killer] gem. +GitLab.com adjusts the memory limits for the [unicorn-worker-killer](https://rubygems.org/gems/unicorn-worker-killer) gem. Base default: @@ -600,13 +624,3 @@ Service discovery: High Performance TCP/HTTP Load Balancer: - [`gitlab-cookbooks` / `gitlab-haproxy` · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab-haproxy) - -[autoscale mode]: https://docs.gitlab.com/runner/configuration/autoscale.html "How Autoscale works" -[runners-post]: https://about.gitlab.com/blog/2016/04/05/shared-runners/ "Shared Runners on GitLab.com" -[GitLab Runner]: https://gitlab.com/gitlab-org/gitlab-runner -[altssh]: https://about.gitlab.com/blog/2016/02/18/gitlab-dot-com-now-supports-an-alternate-git-plus-ssh-port/ "GitLab.com now supports an alternate git+ssh port" -[docker in docker]: https://hub.docker.com/_/docker/ "Docker in Docker at DockerHub" -[mailgun]: https://www.mailgun.com/ "Mailgun website" -[unicorn-worker-killer]: https://rubygems.org/gems/unicorn-worker-killer "unicorn-worker-killer" -[4010]: https://gitlab.com/gitlab-com/infrastructure/issues/4010 "Find a good value for maximum timeout for Shared Runners" -[4070]: https://gitlab.com/gitlab-com/infrastructure/issues/4070 "Configure per-runner timeout for shared-runners-manager-X on GitLab.com" diff --git a/doc/user/group/clusters/index.md b/doc/user/group/clusters/index.md index 224836b20b5..5e6ff980c8b 100644 --- a/doc/user/group/clusters/index.md +++ b/doc/user/group/clusters/index.md @@ -1,13 +1,14 @@ --- type: reference +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Group-level Kubernetes clusters > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/34758) in GitLab 11.6. -## Overview - Similar to [project-level](../../project/clusters/index.md) and [instance-level](../../instance/clusters/index.md) Kubernetes clusters, group-level Kubernetes clusters allow you to connect a Kubernetes cluster to @@ -22,47 +23,43 @@ and troubleshooting applications for your group cluster, see ## RBAC compatibility -For each project under a group with a Kubernetes cluster, GitLab will -create a restricted service account with [`edit` -privileges](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) -in the project namespace. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/29398) in GitLab 11.4. +> - [Project namespace restriction](https://gitlab.com/gitlab-org/gitlab-foss/issues/51716) was introduced in GitLab 11.5. -NOTE: **Note:** -RBAC support was introduced in -[GitLab 11.4](https://gitlab.com/gitlab-org/gitlab-foss/issues/29398), and -Project namespace restriction was introduced in -[GitLab 11.5](https://gitlab.com/gitlab-org/gitlab-foss/issues/51716). +For each project under a group with a Kubernetes cluster, GitLab creates a restricted +service account with [`edit` privileges](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) +in the project namespace. ## Cluster precedence -GitLab will use the project's cluster before using any cluster belonging -to the group containing the project if the project's cluster is available and not disabled. - -In the case of sub-groups, GitLab will use the cluster of the closest ancestor group +If the project's cluster is available and not disabled, GitLab uses the +project's cluster before using any cluster belonging to the group containing +the project. +In the case of sub-groups, GitLab uses the cluster of the closest ancestor group to the project, provided the cluster is not disabled. ## Multiple Kubernetes clusters **(PREMIUM)** -With GitLab Premium, you can associate more than one Kubernetes clusters to your -group. That way you can have different clusters for different environments, -like dev, staging, production, etc. +With [GitLab Premium](https://about.gitlab.com/pricing/premium/), you can associate +more than one Kubernetes cluster to your group, and maintain different clusters +for different environments, such as development, staging, and production. -Add another cluster similar to the first one and make sure to -[set an environment scope](#environment-scopes-premium) that will -differentiate the new cluster from the rest. +When adding another cluster, +[set an environment scope](#environment-scopes-premium) to help +differentiate the new cluster from your other clusters. ## GitLab-managed clusters > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22011) in GitLab 11.5. > - Became [optional](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26565) in GitLab 11.11. -You can choose to allow GitLab to manage your cluster for you. If your cluster is -managed by GitLab, resources for your projects will be automatically created. See the -[Access controls](../../project/clusters/add_remove_clusters.md#access-controls) section for details on which resources will -be created. +You can choose to allow GitLab to manage your cluster for you. If GitLab manages +your cluster, resources for your projects will be automatically created. See the +[Access controls](../../project/clusters/add_remove_clusters.md#access-controls) +section for details on which resources GitLab creates for you. -For clusters not managed by GitLab, project-specific resources will not be created -automatically. If you are using [Auto DevOps](../../../topics/autodevops/index.md) +For clusters not managed by GitLab, project-specific resources won't be created +automatically. If you're using [Auto DevOps](../../../topics/autodevops/index.md) for deployments with a cluster not managed by GitLab, you must ensure: - The project's deployment service account has permissions to deploy to @@ -72,8 +69,8 @@ for deployments with a cluster not managed by GitLab, you must ensure: `KUBE_NAMESPACE` directly is discouraged. NOTE: **Note:** -If you [install applications](#installing-applications) on your cluster, GitLab will create -the resources required to run these even if you have chosen to manage your own cluster. +If you [install applications](#installing-applications) on your cluster, GitLab creates +the resources required to run them even if you choose to manage your own cluster. ### Clearing the cluster cache @@ -86,7 +83,8 @@ your cluster, which can cause deployment jobs to fail. To clear the cache: -1. Navigate to your group’s **Kubernetes** page, and select your cluster. +1. Navigate to your group’s **{cloud-gear}** **Kubernetes** page, + and select your cluster. 1. Expand the **Advanced settings** section. 1. Click **Clear cluster cache**. @@ -105,17 +103,17 @@ The domain should have a wildcard DNS configured to the Ingress IP address. When adding more than one Kubernetes cluster to your project, you need to differentiate them with an environment scope. The environment scope associates clusters with -[environments](../../../ci/environments.md) similar to how the -[environment-specific variables](../../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables) +[environments](../../../ci/environments/index.md) similar to how the +[environment-specific variables](../../../ci/variables/README.md#limit-the-environment-scopes-of-environment-variables) work. While evaluating which environment matches the environment scope of a -cluster, [cluster precedence](#cluster-precedence) will take -effect. The cluster at the project level will take precedence, followed +cluster, [cluster precedence](#cluster-precedence) takes +effect. The cluster at the project level takes precedence, followed by the closest ancestor group, followed by that groups' parent and so on. -For example, let's say we have the following Kubernetes clusters: +For example, if your project has the following Kubernetes clusters: | Cluster | Environment scope | Where | | ---------- | ------------------- | ----------| @@ -151,23 +149,22 @@ deploy to production: url: https://example.com/ ``` -The result will then be: +The result is: -- The Project cluster will be used for the `test` job. -- The Staging cluster will be used for the `deploy to staging` job. -- The Production cluster will be used for the `deploy to production` job. +- The Project cluster is used for the `test` job. +- The Staging cluster is used for the `deploy to staging` job. +- The Production cluster is used for the `deploy to production` job. ## Cluster environments **(PREMIUM)** -For a consolidated view of which CI [environments](../../../ci/environments.md) +For a consolidated view of which CI [environments](../../../ci/environments/index.md) are deployed to the Kubernetes cluster, see the documentation for [cluster environments](../../clusters/environments.md). ## Security of Runners For important information about securely configuring GitLab Runners, see -[Security of -Runners](../../project/clusters/add_remove_clusters.md#security-of-gitlab-runners) +[Security of Runners](../../project/clusters/add_remove_clusters.md#security-of-gitlab-runners) documentation for project-level clusters. ## More information diff --git a/doc/user/group/contribution_analytics/index.md b/doc/user/group/contribution_analytics/index.md index 1bbc40a14a4..03f0ad6ad1c 100644 --- a/doc/user/group/contribution_analytics/index.md +++ b/doc/user/group/contribution_analytics/index.md @@ -1,7 +1,10 @@ --- type: reference +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- - # Contribution Analytics **(STARTER)** > - Introduced in [GitLab Starter](https://about.gitlab.com/pricing/) 8.3. diff --git a/doc/user/group/epics/img/epic_view_v12.3.png b/doc/user/group/epics/img/epic_view_v12.3.png Binary files differdeleted file mode 100644 index 79758cf3d52..00000000000 --- a/doc/user/group/epics/img/epic_view_v12.3.png +++ /dev/null diff --git a/doc/user/group/epics/img/epic_view_v13.0.png b/doc/user/group/epics/img/epic_view_v13.0.png Binary files differnew file mode 100644 index 00000000000..b25a91d318a --- /dev/null +++ b/doc/user/group/epics/img/epic_view_v13.0.png diff --git a/doc/user/group/epics/index.md b/doc/user/group/epics/index.md index 024f346ad47..d53acaf502a 100644 --- a/doc/user/group/epics/index.md +++ b/doc/user/group/epics/index.md @@ -5,25 +5,14 @@ type: reference, howto # Epics **(PREMIUM)** > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.2. -> - In [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/37081), single-level Epics were moved to the Premium tier. +> - Single-level Epics [were moved](https://gitlab.com/gitlab-org/gitlab/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. Epics let you manage your portfolio of projects more efficiently and with less effort by tracking groups of issues that share a theme, across projects and milestones. -## Relationships between epics and issues - -The possible relationships between epics and issues are: - -- An epic is the parent of one or more issues. -- An epic is the parent of one or more child epics. For details see [Multi-level child epics](#multi-level-child-epics-ultimate). **(ULTIMATE)** - -```mermaid -graph TD - Parent_epic --> Issue1 - Parent_epic --> Child_epic - Child_epic --> Issue2 -``` +<!-- Possibly swap this file with one of a single epic --> + ## Use cases @@ -31,75 +20,37 @@ graph TD - Track when the work for the group of issues is targeted to begin, and when it's targeted to end. - Discuss and collaborate on feature ideas and scope at a high level. - - -## Creating an epic - -A paginated list of epics is available in each group from where you can create -a new epic. The list of epics includes also epics from all subgroups of the -selected group. From your group page: - -1. Go to **Epics**. -1. Click **New epic**. -1. Enter a descriptive title and click **Create epic**. - -You will be taken to the new epic where can edit the following details: - -- Title -- Description -- Start date -- Due date -- Labels - -An epic's page contains the following tabs: - -- **Epics and Issues**: epics and issues added to this epic. Child epics, and their issues, are shown in a tree view. - - Click on the <kbd>></kbd> beside a parent epic to reveal the child epics and issues. - - Hover over the total counts to see a breakdown of open and closed items. -- **Roadmap**: a roadmap view of child epics which have start and due dates. - - - -## Adding an issue to an epic - -You can add an existing issue to an epic, or, from an epic's page, create a new issue that's automatically added to the epic. +## Manage epics -### Adding an existing issue to an epic +To learn what you can do with an epic, see [Manage epics](manage_epics.md). Possible actions include: -Existing issues that belong to a project in an epic's group, or any of the epic's -subgroups, are eligible to be added to the epic. Newly added issues appear at the top of the list of issues in the **Epics and Issues** tab. +- [Create an epic](manage_epics.md#create-an-epic) +- [Bulk-edit epics](manage_epics.md#bulk-edit-epics) +- [Delete an epic](manage_epics.md#delete-an-epic) +- [Close an epic](manage_epics.md#close-an-epic) +- [Reopen a closed epic](manage_epics.md#reopen-a-closed-epic) +- [Go to an epic from an issue](manage_epics.md#go-to-an-epic-from-an-issue) +- [Search for an epic from epics list page](manage_epics.md#search-for-an-epic-from-epics-list-page) +- [Make an epic confidential](manage_epics.md#make-an-epic-confidential) +- [Manage issues assigned to an epic](manage_epics.md#manage-issues-assigned-to-an-epic) +- [Manage multi-level child epics **(ULTIMATE)**](manage_epics.md#manage-multi-level-child-epics-ultimate) -An epic contains a list of issues and an issue can be associated with at most -one epic. When you add an issue that's already linked to an epic, -the issue is automatically unlinked from its current parent. - -To add an issue to an epic: - -1. Click **Add an issue**. -1. Identify the issue to be added, using either of the following methods: - - Paste the link of the issue. - - Search for the desired issue by entering part of the issue's title, then selecting the desired match. ([From GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/9126)) - - If there are multiple issues to be added, press <kbd>Spacebar</kbd> and repeat this step. -1. Click **Add**. - -### Creating an issue from an epic - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5419) in GitLab 12.7. - -Creating an issue from an epic enables you to maintain focus on the broader context of the epic while dividing work into smaller parts. +## Relationships between epics and issues -To create an issue from an epic: +The possible relationships between epics and issues are: -1. On the epic's page, under **Epics and Issues**, click the arrow next to **Add an issue** and select **Create new issue**. -1. Under **Title**, enter the title for the new issue. -1. From the **Project** dropdown, select the project in which the issue should be created. -1. Click **Create issue**. +- An epic is the parent of one or more issues. +- An epic is the parent of one or more child epics. For details see [Multi-level child epics](#multi-level-child-epics-ultimate). **(ULTIMATE)** -To remove an issue from an epic: +```mermaid +graph TD + Parent_epic --> Issue1 + Parent_epic --> Child_epic + Child_epic --> Issue2 +``` -1. Click on the <kbd>x</kbd> button in the epic's issue list. -1. Click **Remove** in the **Remove issue** warning message. +See [Manage issues assigned to an epic](manage_epics.md#manage-issues-assigned-to-an-epic) for steps +to add an issue to an epic, reorder issues, move issues between epics, or promote an issue to an epic. ## Issue health status in Epic tree **(ULTIMATE)** @@ -118,28 +69,15 @@ This feature comes with a feature flag enabled by default. For steps to disable > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8333) in GitLab Ultimate 11.7. -Any epic that belongs to a group, or subgroup of the parent epic's group, is -eligible to be added. New child epics appear at the top of the list of epics in the **Epics and Issues** tab. +Any epic that belongs to a group, or subgroup of the parent epic's group, is eligible to be added. +New child epics appear at the top of the list of epics in the **Epics and Issues** tab. When you add an epic that's already linked to a parent epic, the link to its current parent is removed. -An epic can have multiple child epics with -the maximum depth being 5. - -To add a child epic to an epic: - -1. Click **Add an epic**. -1. Identify the epic to be added, using either of the following methods: - - Paste the link of the epic. - - Search for the desired issue by entering part of the epic's title, then selecting the desired match. ([From GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/9126)) +An epic can have multiple child epics up to the maximum depth of five. - If there are multiple epics to be added, press <kbd>Spacebar</kbd> and repeat this step. -1. Click **Add**. - -To remove a child epic from a parent epic: - -1. Click on the <kbd>x</kbd> button in the parent epic's list of epics. -1. Click **Remove** in the **Remove epic** warning message. +See [Manage multi-level child epics](manage_epics.md#manage-multi-level-child-epics-ultimate) for +steps to create, move, reorder, or delete child epics. ## Start date and due date @@ -197,137 +135,6 @@ have a [start or due date](#start-date-and-due-date), a  ---- - -## Reordering issues and child epics - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9367) in GitLab 12.5. - -New issues and child epics are added to the top of their respective lists in the **Epics and Issues** tab. You can reorder the list of issues and the list of child epics. Issues and child epics cannot be intermingled. - -To reorder issues assigned to an epic: - -1. Go to the **Epics and Issues** tab. -1. Drag and drop issues into the desired order. - -To reorder child epics assigned to an epic: - -1. Go to the **Epics and Issues** tab. -1. Drag and drop epics into the desired order. - -## Updating epics - -### Using bulk editing - -To apply labels across multiple epics: - -1. Go to the Epics list. -1. Click **Edit epics**. - - Checkboxes will appear beside each epic. - - A sidebar on the right-hand side will appear, with an editable field for labels. -1. Check the checkbox beside each epic to be edited. -1. Select the desired labels. -1. Click **Update all**. - - - -## Deleting an epic - -NOTE: **Note:** -To delete an epic, you need to be an [Owner](../../permissions.md#group-members-permissions) of a group/subgroup. - -When inside a single epic view, click the **Delete** button to delete the epic. -A modal will pop-up to confirm your action. - -Deleting an epic releases all existing issues from their associated epic in the -system. - -## Closing and reopening epics - -### Using buttons - -Whenever you decide that there is no longer need for that epic, -close the epic using the close button: - - - -You can always reopen it using the reopen button. - - - ---- - -### Using quick actions - -You can close or reopen an epic using [Quick actions](../../project/quick_actions.md) - -## Navigating to an epic from an issue - -If an issue belongs to an epic, you can navigate to the containing epic with the -link in the issue sidebar. - - - ---- - -## Promoting an issue to an epic - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3777) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.6. -> - In [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/37081), it was moved to the Premium tier. - -If you have [permissions](../../permissions.md) to close an issue and create an -epic in the parent group, you can promote an issue to an epic with the `/promote` -[quick action](../../project/quick_actions.md#quick-actions-for-issues-merge-requests-and-epics). -Only issues from projects that are in groups can be promoted. When attempting to promote a confidential -issue, a warning will display. Promoting a confidential issue to an epic will make all information -related to the issue public as epics are public to group members. - -When the quick action is executed: - -- An epic is created in the same group as the project of the issue. -- Subscribers of the issue are notified that the epic was created. - -The following issue metadata will be copied to the epic: - -- Title, description, activity/comment thread. -- Upvotes/downvotes. -- Participants. -- Group labels that the issue already has. - -## Searching for an epic from epics list page - -> - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.5. -> - In [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/37081), it was moved to the Premium tier. - -You can search for an epic from the list of epics using filtered search bar (similar to -that of Issues and Merge Requests) based on following parameters: - -- Title or description -- Author name / username -- Labels - - - -To search, go to the list of epics and click on the field **Search or filter results**. -It will display a dropdown menu, from which you can add an author. You can also enter plain -text to search by epic title or description. When done, press <kbd>Enter</kbd> on your -keyboard to filter the list. - -You can also sort epics list by: - -- Created date -- Last updated -- Start date -- Due date - -Each option contains a button that can toggle the order between **Ascending** and **Descending**. -The sort option and order is saved and used wherever you browse epics, including the -[Roadmap](../roadmap/index.md). - - - ---- - ## Permissions If you have access to view an epic and have access to view an issue already diff --git a/doc/user/group/epics/manage_epics.md b/doc/user/group/epics/manage_epics.md new file mode 100644 index 00000000000..50eb0c64f4b --- /dev/null +++ b/doc/user/group/epics/manage_epics.md @@ -0,0 +1,302 @@ +--- +type: howto +--- + +<!-- When adding a new section here, remember to mention it in index.md#manage-epics --> + +# Manage epics **(PREMIUM)** + +This page collects instructions for all the things you can do with [epics](index.md) or in relation +to them. + +## Create an epic + +A paginated list of epics is available in each group from where you can create +a new epic. The list of epics includes also epics from all subgroups of the +selected group. From your group page: + +1. Go to **Epics**. +1. Click **New epic**. +1. Enter a descriptive title. +1. Click **Create epic**. + +You will be taken to the new epic where can edit the following details: + +- Title +- Description +- Start date +- Due date +- Labels + +An epic's page contains the following tabs: + +- **Epics and Issues**: epics and issues added to this epic. Child epics, and their issues, are shown in a tree view. + - Click the <kbd>></kbd> beside a parent epic to reveal the child epics and issues. + - Hover over the total counts to see a breakdown of open and closed items. +- **Roadmap**: a roadmap view of child epics which have start and due dates. + + + +## Bulk-edit epics + +You can edit multiple epics at once. For example, to apply labels to multiple epics: + +1. Go to the Epics list. +1. Click **Edit epics**. + - Checkboxes appear next to each epic. + - A sidebar on the right-hand side appears with an editable field for labels. +1. Select the checkbox next to each epic to be edited. +1. Select the labels you want. +1. Click **Update all**. + + + +## Delete an epic + +NOTE: **Note:** +To delete an epic, you need to be an [Owner](../../permissions.md#group-members-permissions) of a group/subgroup. + +When editing the description of an epic, click the **Delete** button to delete the epic. +A modal appears to confirm your action. + +Deleting an epic releases all existing issues from their associated epic in the system. + +## Close an epic + +Whenever you decide that there is no longer need for that epic, +close the epic by: + +- Clicking the **Close epic** button. + +  + +- Using a [quick action](../../project/quick_actions.md). + +## Reopen a closed epic + +You can reopen an epic that was closed by: + +- Clicking the **Reopen epic** button. + +  + +- Using a [quick action](../../project/quick_actions.md). + +## Go to an epic from an issue + +If an issue belongs to an epic, you can navigate to the containing epic with the +link in the issue sidebar. + + + +## Search for an epic from epics list page + +> - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.5. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/37081) to the [Premium](https://about.gitlab.com/pricing/) tier in GitLab 12.8. + +You can search for an epic from the list of epics using filtered search bar (similar to +that of Issues and Merge Requests) based on following parameters: + +- Title or description +- Author name / username +- Labels + + + +To search, go to the list of epics and click the field **Search or filter results**. +It will display a dropdown menu, from which you can add an author. You can also enter plain +text to search by epic title or description. When done, press <kbd>Enter</kbd> on your +keyboard to filter the list. + +You can also sort epics list by: + +- Created date +- Last updated +- Start date +- Due date + +Each option contains a button that can toggle the order between **Ascending** and **Descending**. +The sort option and order is saved and used wherever you browse epics, including the +[Roadmap](../roadmap/index.md). + + + +## Make an epic confidential + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213068) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-confidential-epics-premium-only). **(PREMIUM ONLY)** + +When you're creating an epic, you can make it confidential by selecting the **Make this epic +confidential** checkbox. + +### Enable Confidential Epics **(PREMIUM ONLY)** + +The Confidential Epics feature is under development and not ready for production use. It's deployed behind a +feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:confidential_epics) +``` + +To disable it: + +```ruby +Feature.disable(:confidential_epics) +``` + +## Manage issues assigned to an epic + +### Add an issue to an epic + +You can add an existing issue to an epic, or, create a new issue that's +automatically added to the epic. + +#### Add an existing issue to an epic + +Existing issues that belong to a project in an epic's group, or any of the epic's +subgroups, are eligible to be added to the epic. Newly added issues appear at the top of the list of +issues in the **Epics and Issues** tab. + +An epic contains a list of issues and an issue can be associated with at most one epic. +When you add an issue that's already linked to an epic, the issue is automatically unlinked from its +current parent. + +To add an issue to an epic: + +1. Click the **Add** dropdown button. +1. Click **Add an issue**. +1. Identify the issue to be added, using either of the following methods: + - Paste the link of the issue. + - Search for the desired issue by entering part of the issue's title, then selecting the desired + match (introduced in [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/9126)). + + If there are multiple issues to be added, press <kbd>Spacebar</kbd> and repeat this step. +1. Click **Add**. + +#### Create an issue from an epic + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5419) in GitLab 12.7. + +Creating an issue from an epic enables you to maintain focus on the broader context of the epic +while dividing work into smaller parts. + +To create an issue from an epic: + +1. On the epic's page, under **Epics and Issues**, click the **Add** dropdown button and select + **Create new issue**. +1. Under **Title**, enter the title for the new issue. +1. From the **Project** dropdown, select the project in which the issue should be created. +1. Click **Create issue**. + +To remove an issue from an epic: + +1. Click the <kbd>x</kbd> button in the epic's issue list. +1. Click **Remove** in the **Remove issue** warning message. + +### Reorder issues assigned to an epic + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9367) in GitLab 12.5. + +New issues are added to the top of their list in the **Epics and Issues** tab. +You can reorder the list of issues. Issues and child epics cannot be intermingled. + +To reorder issues assigned to an epic: + +1. Go to the **Epics and Issues** tab. +1. Drag and drop issues into the desired order. + +To reorder child epics assigned to an epic: + +1. Go to the **Epics and Issues** tab. +1. Drag and drop epics into the desired order. + +### Move issues between epics **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. + +New issues are added to the top of their list in the **Epics and Issues** +tab. You can move issues from one epic to another. Issues and child epics cannot be intermingled. + +To move an issue to another epic: + +1. Go to the **Epics and Issues** tab. +1. Drag and drop issues into the desired parent epic. + +### Promote an issue to an epic + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3777) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.6. +> - In [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/37081), it was moved to the Premium tier. + +If you have [permissions](../../permissions.md) to close an issue and create an +epic in the parent group, you can promote an issue to an epic with the `/promote` +[quick action](../../project/quick_actions.md#quick-actions-for-issues-merge-requests-and-epics). +Only issues from projects that are in groups can be promoted. When attempting to promote a confidential +issue, a warning will display. Promoting a confidential issue to an epic will make all information +related to the issue public as epics are public to group members. + +When the quick action is executed: + +- An epic is created in the same group as the project of the issue. +- Subscribers of the issue are notified that the epic was created. + +The following issue metadata will be copied to the epic: + +- Title, description, activity/comment thread. +- Upvotes/downvotes. +- Participants. +- Group labels that the issue already has. + +## Manage multi-level child epics **(ULTIMATE)** + +### Add a child epic to an epic + +To add a child epic to an epic: + +1. Click the **Add** dropdown button. +1. Click **Add an epic**. +1. Identify the epic to be added, using either of the following methods: + - Paste the link of the epic. + - Search for the desired issue by entering part of the epic's title, then selecting the desired match (introduced in [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/9126)). + + If there are multiple epics to be added, press <kbd>Spacebar</kbd> and repeat this step. +1. Click **Add**. + +### Move child epics between epics + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. + +New child epics are added to the top of their list in the **Epics and Issues** tab. +You can move child epics from one epic to another. +When you add an epic that's already linked to a parent epic, the link to its current parent is removed. +Issues and child epics cannot be intermingled. + +To move child epics to another epic: + +1. Go to the **Epics and Issues** tab. +1. Drag and drop epics into the desired parent epic. + +### Reorder child epics assigned to an epic + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9367) in GitLab 12.5. + +New child epics are added to the top of their list in the **Epics and Issues** tab. +You can reorder the list of child epics. Issues and child epics cannot be intermingled. + +To reorder child epics assigned to an epic: + +1. Go to the **Epics and Issues** tab. +1. Drag and drop epics into the desired order. + +### Remove a child epic from a parent epic + +To remove a child epic from a parent epic: + +1. Click on the <kbd>x</kbd> button in the parent epic's list of epics. +1. Click **Remove** in the **Remove epic** warning message. diff --git a/doc/user/group/index.md b/doc/user/group/index.md index b819e3e971e..f36f3b3fd4f 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -196,6 +196,9 @@ To change this setting for a specific group: To change this setting globally, see [Default branch protection](../admin_area/settings/visibility_and_access_controls.md#default-branch-protection). +NOTE: **Note:** +In [GitLab Premium or higher](https://about.gitlab.com/pricing/), GitLab administrators can choose to [disable group owners from updating the default branch protection](../admin_area/settings/visibility_and_access_controls.md#disable-group-owners-from-updating-default-branch-protection-premium-only). + ## Add projects to a group There are two different ways to add a new project to a group: @@ -211,8 +214,8 @@ There are two different ways to add a new project to a group: ### Default project-creation level -> - [Introduced][ee-2534] in [GitLab Premium][ee] 10.5. -> - Brought to [GitLab Starter][ee] in 10.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2534) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. +> - Brought to [GitLab Starter](https://about.gitlab.com/pricing/) in 10.7. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25975) to [GitLab Core](https://about.gitlab.com/pricing/) in 11.10. By default, [Developers and Maintainers](../permissions.md#group-members-permissions) can create projects under a group. @@ -308,7 +311,7 @@ See [the GitLab Enterprise Edition documentation](../../integration/ldap.md) for ## Epics **(ULTIMATE)** -> Introduced in [GitLab Ultimate][ee] 10.2. +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.2. Epics let you manage your portfolio of projects more efficiently and with less effort by tracking groups of issues that share a theme, across projects and @@ -593,6 +596,29 @@ For performance reasons, we may delay the update up to 1 hour and 30 minutes. If your namespace shows `N/A` as the total storage usage, you can trigger a recalculation by pushing a commit to any project in that namespace. +#### Group push rules **(STARTER)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34370) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. + +Group push rules allow group maintainers to set +[push rules](../../push_rules/push_rules.md) for newly created projects within the specific group. + +To configure push rules for a group, navigate to **{push-rules}** on the group's +sidebar. + +When set, new subgroups have push rules set for them based on either: + +- The closest parent group with push rules defined. +- Push rules set at the instance level, if no parent groups have push rules defined. + +##### Enabling the feature + +This feature comes with the `:group_push_rules` feature flag disabled by default. It can be enabled for specific group using feature flag [API endpoint](../../api/features.md#set-or-create-a-feature) or by GitLab administrator with Rails console access by running: + +```ruby +Feature.enable(:group_push_rules) +``` + ### Maximum artifacts size **(CORE ONLY)** For information about setting a maximum artifact size for a group, see @@ -623,6 +649,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -[ee]: https://about.gitlab.com/pricing/ -[ee-2534]: https://gitlab.com/gitlab-org/gitlab/issues/2534 diff --git a/doc/user/group/insights/index.md b/doc/user/group/insights/index.md index edbb85962ed..cffbc013e66 100644 --- a/doc/user/group/insights/index.md +++ b/doc/user/group/insights/index.md @@ -1,5 +1,9 @@ --- type: reference, howto +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Insights **(ULTIMATE)** diff --git a/doc/user/group/issues_analytics/index.md b/doc/user/group/issues_analytics/index.md index 4477b9bb1e6..df96f2626e1 100644 --- a/doc/user/group/issues_analytics/index.md +++ b/doc/user/group/issues_analytics/index.md @@ -1,5 +1,9 @@ --- type: reference +stage: Manage +group: Analytics +To determine the technical writer assigned to the Stage/Group associated with this page, see: + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Issues Analytics **(PREMIUM)** diff --git a/doc/user/group/roadmap/img/roadmap_view_v12_10.png b/doc/user/group/roadmap/img/roadmap_view_v12_10.png Binary files differdeleted file mode 100644 index 69579fd1c1e..00000000000 --- a/doc/user/group/roadmap/img/roadmap_view_v12_10.png +++ /dev/null diff --git a/doc/user/group/roadmap/img/roadmap_view_v13_0.png b/doc/user/group/roadmap/img/roadmap_view_v13_0.png Binary files differnew file mode 100644 index 00000000000..a5b76b84418 --- /dev/null +++ b/doc/user/group/roadmap/img/roadmap_view_v13_0.png diff --git a/doc/user/group/roadmap/index.md b/doc/user/group/roadmap/index.md index 9f068adcd47..6bee552d433 100644 --- a/doc/user/group/roadmap/index.md +++ b/doc/user/group/roadmap/index.md @@ -7,7 +7,8 @@ type: reference > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.5. > - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/198062), Roadmaps were moved to the Premium tier. > - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/5164) and later, the epic bars show epics' title, progress, and completed weight percentage. -> - In [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/6802), and later, milestones appear in Roadmaps. +> - Milestones appear in Roadmaps in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/6802), and later. +> - Feature flag removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29641). Epics and milestones within a group containing **Start date** and/or **Due date** can be visualized in a form of a timeline (that is, a Gantt chart). The Roadmap page @@ -23,7 +24,7 @@ You can click the chevron **{chevron-down}** next to the epic title to expand an On top of the milestone bars, you can see their title. When you hover a milestone bar or title, a popover appears with its title, start date and due date. - + A dropdown menu allows you to show only open or closed epics. By default, all epics are shown. diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index f49dd225146..a3d9a14df10 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -59,7 +59,7 @@ We recommend setting the NameID format to `Persistent` unless using a field (suc With this option enabled, users must use your group's GitLab single sign on URL to be added to the group or be added via SCIM. Users cannot be added manually, and may only access project/group resources via the UI by signing in through the SSO URL. -However, users will not be prompted to log via SSO on each visit. GitLab will check whether a user has authenticated through the SSO link, and will only prompt the user to login via SSO if it has been longer than 7 days. +However, users will not be prompted to log via SSO on each visit. GitLab will check whether a user has authenticated through the SSO link, and will only prompt the user to login via SSO if the session has expired. We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab/issues/9152) in the future. @@ -189,6 +189,9 @@ Once you've set up your identity provider to work with GitLab, you'll need to co  +NOTE: **Note:** +Please note that the certificate [fingerprint algorithm](#additional-setup-options) must be in SHA1. When configuring the identity provider, use a secure [signature algorithm](#additional-setup-options). + ## User access and management Once Group SSO is configured and enabled, users can access the GitLab.com group through the identity provider's dashboard. If [SCIM](scim_setup.md) is configured, please see the [user access and linking setup section on the SCIM page](scim_setup.md#user-access-and-linking-setup). @@ -254,6 +257,9 @@ Set other user attributes and claims according to the [assertions table](#assert ### Okta setup notes +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a demo of the Okta SAML setup including SCIM, see [Demo: Okta Group SAML & SCIM setup](https://youtu.be/0ES9HsZq0AQ). + | GitLab Setting | Okta Field | |--------------|----------------| | Identifier | Audience URI | @@ -297,7 +303,7 @@ GitLab [isn't limited to the SAML providers listed above](#my-identity-provider- | SAML Request Binding | HTTP Redirect | GitLab (the service provider) redirects users to your Identity Provider with a base64 encoded `SAMLRequest` HTTP parameter. | | SAML Response Binding | HTTP POST | Your Identity Provider responds to users with an HTTP form including the `SAMLResponse`, which a user's browser submits back to GitLab. | | Sign SAML Response | Yes | We require this to prevent tampering. | -| X509 Certificate in response | Yes | This is used to sign the response and checked against the provided fingerprint. | +| X.509 Certificate in response | Yes | This is used to sign the response and checked against the provided fingerprint. | | Fingerprint Algorithm | SHA-1 | We need a SHA-1 hash of the certificate used to sign the SAML Response. | | Signature Algorithm | SHA-1/SHA-256/SHA-384/SHA-512 | Also known as the Digest Method, this can be specified in the SAML response. It determines how a response is signed. | | Encrypt SAML Assertion | No | TLS is used between your Identity Provider, the user's browser, and GitLab. | diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index e333fd19c1b..f66c8a788b6 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -12,25 +12,28 @@ that group is synchronized between GitLab and the identity provider. GitLab's [SCIM API](../../../api/scim.md) implements part of [the RFC7644 protocol](https://tools.ietf.org/html/rfc7644). +## Features + Currently, the following actions are available: -- CREATE -- UPDATE -- DELETE (deprovisioning) +- Create users +- Update users (Azure only) +- Deactivate users The following identity providers are supported: - Azure +- Okta ## Requirements -- [Group SSO](index.md) must be configured. +- [Group Single Sign-On](index.md) must be configured. ## GitLab configuration -Once [Single sign-on](index.md) has been configured, we can: +Once [Group Single Sign-On](index.md) has been configured, we can: -1. Navigate to the group and click **Settings > SAML SSO**. +1. Navigate to the group and click **Administration > SAML SSO**. 1. Click on the **Generate a SCIM token** button. 1. Save the token and URL so they can be used in the next step. @@ -38,9 +41,12 @@ Once [Single sign-on](index.md) has been configured, we can: ## Identity Provider configuration -### Azure +- [Azure](#azure-configuration-steps) +- [Okta](#okta-configuration-steps) + +### Azure configuration steps -The SAML application that was created during [Single sign-on](index.md) setup now needs to be set up for SCIM. +The SAML application that was created during [Single sign-on](index.md) setup for [Azure](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/configure-single-sign-on-non-gallery-applications) now needs to be set up for SCIM. 1. Check the configuration for your GitLab SAML app and ensure that **Name identifier value** (NameID) points to `user.objectid` or another unique identifier. This will match the `extern_uid` used on GitLab. @@ -109,6 +115,43 @@ bottom of the **Provisioning** screen, together with a link to the audit logs. CAUTION: **Warning:** Once synchronized, changing the field mapped to `id` and `externalId` will likely cause provisioning errors, duplicate users, and prevent existing users from accessing the GitLab group. +### Okta configuration steps + +The SAML application that was created during [Single sign-on](index.md#okta-setup-notes) setup for [Okta](https://developer.okta.com/docs/guides/saml-application-setup/overview/) now needs to be set up for SCIM. +Before proceeding, be sure to complete the [GitLab configuration](#gitlab-configuration) process. + +1. Sign in to Okta. +1. If you see an **Admin** button in the top right, click the button. This will + ensure you are in the Admin area. + + TIP: **Tip:** If you're using the Developer Console, click **Developer Console** in the top + bar and select **Classic UI**. Otherwise, you may not see the buttons described + in the following steps: + +1. In the **Application** tab, click **Add Application**. +1. Search for **GitLab**, find and click on the 'GitLab' application. +1. On the GitLab application overview page, click **Add**. +1. Under **Application Visibility** select both check boxes. Currently the GitLab application does not support SAML authentication so the icon should not be shown to users. +1. Click **Done** to finish adding the application. +1. In the **Provisioning** tab, click **Configure API integration**. +1. Select **Enable API integration**. + - For **Base URL** enter the URL obtained from the GitLab SCIM configuration page + - For **API Token** enter the SCIM token obtained from the GitLab SCIM configuration page +1. Click 'Test API Credentials' to verify configuration. +1. Click **Save** to apply the settings. +1. After saving the API integration details, new settings tabs will appear on the left. Choose **To App**. +1. Click **Edit**. +1. Check the box to **Enable** for both **Create Users** and **Deactivate Users**. +1. Click **Save**. +1. Assign users in the **Assignments** tab. Assigned users will be created and + managed in your GitLab group. + +#### Okta Known Issues + +The Okta GitLab application currently only supports SCIM. Continue +using the separate Okta [SAML SSO](index.md) configuration along with the new SCIM +application described above. + ## User access and linking setup As long as [Group SAML](index.md) has been configured, prior to turning on sync, existing GitLab.com users can link to their accounts in one of the following ways, before synchronization is active: @@ -135,7 +178,9 @@ Upon the next sync, the user will be deprovisioned, which means that the user wi This section contains possible solutions for problems you might encounter. -### How do I verify my SCIM configuration is correct? +### Azure + +#### How do I verify my SCIM configuration is correct? Review the following: @@ -148,11 +193,11 @@ Review the following SCIM parameters for sensible values: - `displayName` - `emails[type eq "work"].value` -### Testing Azure connection: invalid credentials +#### Testing Azure connection: invalid credentials When testing the connection, you may encounter an error: **You appear to have entered invalid credentials. Please confirm you are using the correct information for an administrative account**. If `Tenant URL` and `secret token` are correct, check whether your group path contains characters that may be considered invalid JSON primitives (such as `.`). Removing such characters from the group path typically resolves the error. -### Azure: (Field) can't be blank sync error +#### Azure: (Field) can't be blank sync error When checking the Audit Logs for the Provisioning, you can sometimes see the error `Namespace can't be blank, Name can't be blank, and User can't be blank.` @@ -165,14 +210,7 @@ As a workaround, try an alternate mapping: 1. Delete the `name.formatted` target attribute entry. 1. Change the `displayName` source attribute to have `name.formatted` target attribute. -### Message: "SAML authentication failed: Email has already been taken" - -This message may be caused by the following: - -- Existing users have not yet signed into the new app. -- The identity provider attempts to create a new user account in GitLab with an email address that already exists in GitLab.com. - -### How do I diagnose why a user is unable to sign in +#### How do I diagnose why a user is unable to sign in The **Identity** (`extern_uid`) value stored by GitLab is updated by SCIM whenever `id` or `externalId` changes. Users won't be able to sign in unless the GitLab Identity (`extern_uid`) value matches the `NameId` sent by SAML. @@ -180,7 +218,7 @@ This value is also used by SCIM to match users on the `id`, and is updated by SC It is important that this SCIM `id` and SCIM `externalId` are configured to the same value as the SAML `NameId`. SAML responses can be traced using [debugging tools](./index.md#saml-debugging-tools), and any errors can be checked against our [SAML troubleshooting docs](./index.md#troubleshooting). -### How do I verify user's SAML NameId matches the SCIM externalId +#### How do I verify user's SAML NameId matches the SCIM externalId Group owners can see the list of users and the `externalId` stored for each user in the group SAML SSO Settings page. @@ -194,7 +232,7 @@ curl 'https://example.gitlab.com/api/scim/v2/groups/GROUP_NAME/Users?startIndex= To see how this compares to the value returned as the SAML NameId, you can have the user use a [SAML Tracer](index.md#saml-debugging-tools). -### Update or fix mismatched SCIM externalId and SAML NameId +#### Update or fix mismatched SCIM externalId and SAML NameId Whether the value was changed or you need to map to a different field, ensure `id`, `externalId`, and `NameId` all map to the same field. @@ -220,7 +258,7 @@ curl --verbose --request PATCH 'https://gitlab.com/api/scim/v2/groups/YOUR_GROUP It is important not to update these to incorrect values, since this will cause users to be unable to sign in. It is also important not to assign a value to the wrong user, as this would cause users to get signed into the wrong account. -### I need to change my SCIM app +#### I need to change my SCIM app Individual users can follow the instructions in the ["SAML authentication failed: User has already been taken"](./index.md#i-need-to-change-my-saml-app) section. diff --git a/doc/user/group/settings/img/export_panel.png b/doc/user/group/settings/img/export_panel.png Binary files differnew file mode 100644 index 00000000000..2a987f04e35 --- /dev/null +++ b/doc/user/group/settings/img/export_panel.png diff --git a/doc/user/group/settings/import_export.md b/doc/user/group/settings/import_export.md new file mode 100644 index 00000000000..3f14fea673b --- /dev/null +++ b/doc/user/group/settings/import_export.md @@ -0,0 +1,98 @@ +# Group Import/Export + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2888) in GitLab 13.0 as an experimental feature. May change in future releases. + +Existing groups running on any GitLab instance or GitLab.com can be exported with all their related data and moved to a +new GitLab instance. + +The **GitLab import/export** button is displayed if the group import option in enabled. + +See also: + +- [Group Import/Export API](../../../api/group_import_export.md) +- [Project Import/Export](../../project/settings/import_export.md) +- [Project Import/Export API](../../../api/project_import_export.md) + +To enable GitLab import/export: + +1. Navigate to **{admin}** **Admin Area >** **{settings}** **Settings > Visibility and access controls**. +1. Scroll to **Import sources** +1. Enable desired **Import sources** + +## Important Notes + +Note the following: + +- Exports are stored in a temporary [shared directory](../../../development/shared_files.md) and are deleted every 24 hours by a specific worker. +- To preserve group-level relationships from imported projects, run the Group Import/Export first, to allow projects to +be imported into the desired group structure. +- Imported groups are given a `private` visibility level, unless imported into a parent group. +- If imported into a parent group, subgroups will inherit the same level of visibility unless otherwise restricted. +- To preserve the member list and their respective permissions on imported groups, review the users in these groups. Make +sure these users exist before importing the desired groups. + +### Exported Contents + +The following items will be exported: + +- Milestones +- Labels +- Boards and Board Lists +- Badges +- Subgroups (including all the aforementioned data) +- Epics +- Events + +The following items will NOT be exported: + +- Projects +- Runners token +- SAML discovery tokens + +NOTE: **Note:** +For more details on the specific data persisted in a group export, see the +[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/import_export/group/import_export.yml) file. + +## Exporting a Group + +1. Navigate to your group's homepage. + +1. Click **{settings}** **Settings** in the sidebar. + +1. In the **Advanced** section, click the **Export Group** button. + +  + +1. Once the export is generated, you should receive an e-mail with a link to the [exported contents](#exported-contents) + in a compressed tar archive, with contents in JSON format. + +1. Alternatively, you can come back to the project settings and download the + file from there by clicking **Download export**, or generate a new file by clicking **Regenerate export**. + +### Between CE and EE + +You can export groups from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) and vice versa. + +If you're exporting a group from the Enterprise Edition to the Community Edition, you may lose data that is retained only in the Enterprise Edition. For more information, see [downgrading from EE to CE](../../../README.md). + +## Version history + +GitLab can import bundles that were exported from a different GitLab deployment. +This ability is limited to two previous GitLab [minor](../../../policy/maintenance.md#versioning) +releases, which is similar to our process for [Security Releases](../../../policy/maintenance.md#security-releases). + +For example: + +| Current version | Can import bundles exported from | +|-----------------|----------------------------------| +| 13.0 | 13.0, 12.10, 12.9 | +| 13.1 | 13.1, 13.0, 12.10 | + +## Rate Limits + +To help avoid abuse, users are rate limited to: + +| Request Type | Limit | +| ---------------- | ------------------------------ | +| Export | 1 group every 5 minutes | +| Download export | 10 downloads every 10 minutes | diff --git a/doc/user/group/subgroups/index.md b/doc/user/group/subgroups/index.md index e73623a2f0e..49f6ddc3986 100644 --- a/doc/user/group/subgroups/index.md +++ b/doc/user/group/subgroups/index.md @@ -191,11 +191,6 @@ Here's a list of what you can't do with subgroups: with `group`, but can be shared with `group/subgroup02` or `group/subgroup01/subgroup03`. -[ce-2772]: https://gitlab.com/gitlab-org/gitlab-foss/issues/2772 -[permissions]: ../../permissions.md#group-members-permissions -[reserved]: ../../reserved_names.md -[issue]: https://gitlab.com/gitlab-org/gitlab-foss/issues/30472#note_27747600 - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/img/gitlab_snippet.png b/doc/user/img/gitlab_snippet.png Binary files differdeleted file mode 100644 index 718347fc2d4..00000000000 --- a/doc/user/img/gitlab_snippet.png +++ /dev/null diff --git a/doc/user/img/gitlab_snippet_v13_0.png b/doc/user/img/gitlab_snippet_v13_0.png Binary files differnew file mode 100644 index 00000000000..33194c512df --- /dev/null +++ b/doc/user/img/gitlab_snippet_v13_0.png diff --git a/doc/user/img/snippet_clone_button_v13_0.png b/doc/user/img/snippet_clone_button_v13_0.png Binary files differnew file mode 100644 index 00000000000..bf681e7349b --- /dev/null +++ b/doc/user/img/snippet_clone_button_v13_0.png diff --git a/doc/user/incident_management/index.md b/doc/user/incident_management/index.md index 8505afb375e..73d6e0e055d 100644 --- a/doc/user/incident_management/index.md +++ b/doc/user/incident_management/index.md @@ -1,141 +1,112 @@ --- -description: "GitLab - Incident Management. GitLab offers solutions for handling incidents in your applications and services" +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Incident Management GitLab offers solutions for handling incidents in your applications and services, -from setting up an alert with Prometheus, to receiving a notification via a -monitoring tool like Slack, and automatically setting up Zoom calls with your -support team. +from setting up an alert with Prometheus, to receiving a notification through a +monitoring tool like Slack, and [setting up Zoom calls](#zoom-integration-in-issues) with your +support team. Incidents can display [metrics](#embed-metrics-in-incidents-and-issues) +and [logs](#view-logs-from-metrics-panel). -## Configuring incidents **(ULTIMATE)** +## Configure incidents **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4925) in GitLab Ultimate 11.11. -The Incident Management features can be enabled and disabled via your project's -**Settings > Operations > Incidents**. +You can enable or disable Incident Management features in your project's +**{settings}** **Settings > Operations > Incidents**. Issues can be created for +each alert triggered, and separate email notifications can be sent to users with +[Developer permissions](../permissions.md). Appropriately configured alerts include an +[embedded chart](../project/integrations/prometheus.md#embedding-metrics-based-on-alerts-in-incident-issues) +for the query corresponding to the alert. You can also configure GitLab to +[close issues](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate) +when you receive notification that the alert is resolved.  -### Automatically create issues from alerts +### Create issues from alerts -GitLab issues can automatically be created as a result of an alert notification. -An issue created this way will contain the error information to help you further -debug it. +You can create GitLab issues from an alert notification. These issues contain +information about the alerts to help you diagnose the source of the alerts. -### Issue templates - -You can create your own [issue templates](../project/description_templates.md#creating-issue-templates) -that can be [used within Incident Management](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate). - -To select your issue template for use within Incident Management: - -1. Visit your project's **Settings > Operations > Incidents**. +1. Visit your project's **{settings}** **Settings > Operations > Incidents**. +1. Select the **Create an issue** checkbox for GitLab to create an issue from + the incident. 1. Select the template from the **Issue Template** dropdown. + You can create your own [issue templates](../project/description_templates.md#creating-issue-templates) + to [use within Incident Management](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate). +1. Click **Save changes**. -## Alerting +## Notify developers of alerts -GitLab can react to the alerts that your applications and services may be -triggering by automatically creating issues, and alerting developers via email. +GitLab can react to the alerts triggered from your applications and services +by creating issues and alerting developers through email. GitLab sends these emails +to [owners and maintainers](../permissions.md) of the project. They contain details +of the alert, and a link for more information. -The emails will be sent to [owners and maintainers](../permissions.md) of the project and will contain details on the alert as well as a link to see more information. +### Configure Prometheus alerts -### Prometheus alerts +You can set up Prometheus alerts in: -Prometheus alerts can be set up in both: - -- [GitLab-managed Prometheus](../project/integrations/prometheus.md#setting-up-alerts-for-prometheus-metrics) and +- [GitLab-managed Prometheus](../project/integrations/prometheus.md#setting-up-alerts-for-prometheus-metrics) installations. - [Self-managed Prometheus](../project/integrations/prometheus.md#external-prometheus-instances) installations. -### Alert endpoint - -GitLab can accept alerts from any source via a generic webhook receiver. When -you set up the generic alerts integration, a unique endpoint will -be created which can receive a payload in JSON format. - -[Read more on setting this up, including how to customize the payload](../project/integrations/generic_alerts.md). +Prometheus alerts are created by the special Alert Bot user. You can't remove this +user, but it does not count toward your license limit. -### Recovery alerts +### Configure external generic alerts -GitLab can [automatically close issues](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate) -that have been automatically created when you receive notification that the -alert is resolved. +GitLab can accept alerts from any source through a generic webhook receiver. When +[configuring the generic alerts integration](../project/integrations/generic_alerts.md), +GitLab creates a unique endpoint which receives a JSON-formatted, customizable payload. -## Embedded metrics +## Embed metrics in incidents and issues -Metrics can be embedded anywhere where GitLab Markdown is used, for example, -descriptions and comments on issues and merge requests. - -This can be useful for when you're sharing metrics, such as for discussing -an incident or performance issues, so you can output the dashboard directly +You can embed metrics anywhere GitLab Markdown is used, such as descriptions, +comments on issues, and merge requests. Embedding metrics helps you share them +when discussing incidents or performance issues. You can output the dashboard directly into any issue, merge request, epic, or any other Markdown text field in GitLab -by simply [copying and pasting the link to the metrics dashboard](../project/integrations/prometheus.md#embedding-gitlab-managed-kubernetes-metrics). - -TIP: **Tip:** -Both GitLab-hosted and Grafana metrics can also be -[embedded in issue templates](../project/integrations/prometheus.md#embedding-metrics-in-issue-templates). - -### GitLab-hosted metrics +by [copying and pasting the link to the metrics dashboard](../project/integrations/prometheus.md#embedding-gitlab-managed-kubernetes-metrics). -Learn how to embed [GitLab hosted metric charts](../project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown). +You can embed both +[GitLab-hosted metrics](../project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown) and +[Grafana metrics](../project/integrations/prometheus.md#embedding-grafana-charts) +in incidents and issue templates. -#### Context menu +### Context menu From each of the embedded metrics panels, you can access more details -about the data you are viewing from a context menu. - -You can access the context menu by clicking the **{ellipsis_v}** **More actions** -dropdown box above the upper right corner of the panel: - -The options are: +about the data you're viewing from a context menu. You can access the context menu +by clicking the **{ellipsis_v}** **More actions** dropdown box above the +upper right corner of the panel. The options are: -- [View logs](#view-logs) -- [Download CSV](#download-csv) +- [View logs](#view-logs-from-metrics-panel). +- **Download CSV** - Data from embedded charts can be + [downloaded as CSV](../project/integrations/prometheus.md#downloading-data-as-csv). -##### View logs +#### View logs from metrics panel > - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201846) in GitLab Ultimate 12.8. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. -This can be useful if you are triaging an application incident and need to -[explore logs](../project/integrations/prometheus.md#view-logs-ultimate) -from across your application. It also helps you to understand -what is affecting your application's performance and quickly resolve any problems. - -##### Download CSV - -Data from embedded charts can be [downloaded as CSV](../project/integrations/prometheus.md#downloading-data-as-csv). - -### Grafana metrics - -Learn how to embed [Grafana hosted metric charts](../project/integrations/prometheus.md#embedding-grafana-charts). +Viewing logs from a metrics panel can be useful if you're triaging an application +incident and need to [explore logs](../project/integrations/prometheus.md#view-logs-ultimate) +from across your application. These logs help you understand what is affecting +your application's performance and resolve any problems. ## Slack integration -Slack slash commands allow you to control GitLab and view content right inside -Slack, without having to leave it. +Slack slash commands allow you to control GitLab and view GitLab content without leaving Slack. Learn how to [set up Slack slash commands](../project/integrations/slack_slash_commands.md) -and how to [use them](../../integration/slash_commands.md). - -### Slash commands - -Please refer to a list of [available slash commands](../../integration/slash_commands.md) and associated descriptions. - -## Zoom in issues - -In order to communicate synchronously for incidents management, GitLab allows you to -associate a Zoom meeting with an issue. Once you start a Zoom call for a fire-fight, -you need a way to associate the conference call with an issue, so that your team -members can join swiftly without requesting a link. - -Read more how to [add or remove a zoom meeting](../project/issues/associate_zoom_meeting.md). - -### Configuring Incidents - -Incident Management features can be easily enabled & disabled via the Project settings page. Head to Project -> Settings -> Operations -> Incidents. +and how to [use the available slash commands](../../integration/slash_commands.md). -#### Auto-creation +## Zoom integration in issues -You can automatically create GitLab issues from an Alert notification. Issues created this way contain error information to help you debug the error. Appropriately configured alerts include an [embedded chart](../project/integrations/prometheus.md#embedding-metrics-based-on-alerts-in-incident-issues) for the query corresponding to the alert. +GitLab enables you to [associate a Zoom meeting with an issue](../project/issues/associate_zoom_meeting.md) +for synchronous communication during incident management. After starting a Zoom +call for an incident, you can associate the conference call with an issue, so your +team members can join without requesting a link. diff --git a/doc/user/infrastructure/img/terraform_plan_log_v13_0.png b/doc/user/infrastructure/img/terraform_plan_log_v13_0.png Binary files differnew file mode 100644 index 00000000000..c3c6f6b2f8b --- /dev/null +++ b/doc/user/infrastructure/img/terraform_plan_log_v13_0.png diff --git a/doc/user/infrastructure/img/terraform_plan_widget_v13_0.png b/doc/user/infrastructure/img/terraform_plan_widget_v13_0.png Binary files differnew file mode 100644 index 00000000000..62bf4b279b2 --- /dev/null +++ b/doc/user/infrastructure/img/terraform_plan_widget_v13_0.png diff --git a/doc/user/infrastructure/index.md b/doc/user/infrastructure/index.md index a50cdf1cf0e..65237bf24e0 100644 --- a/doc/user/infrastructure/index.md +++ b/doc/user/infrastructure/index.md @@ -1,6 +1,343 @@ -# Infrastructure as Code +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -GitLab can be used to manage infrastructure as code. The following are some examples: +# Infrastructure as code with Terraform and GitLab -- [A generic tutorial for Terraform with GitLab](https://medium.com/@timhberry/terraform-pipelines-in-gitlab-415b9d842596). -- [Terraform at GitLab](https://about.gitlab.com/blog/2019/11/12/gitops-part-2/). +## GitLab managed Terraform State + +[Terraform remote backends](https://www.terraform.io/docs/backends/index.html) +enable you to store the state file in a remote, shared store. GitLab uses the +[Terraform HTTP backend](https://www.terraform.io/docs/backends/types/http.html) +to securely store the state files in local storage (the default) or +[the remote store of your choice](../../administration/terraform_state.md). + +The GitLab managed Terraform state backend can store your Terraform state easily and +securely, and spares you from setting up additional remote resources like +Amazon S3 or Google Cloud Storage. Its features include: + +- Supporting encryption of the state file both in transit and at rest. +- Locking and unlocking state. +- Remote Terraform plan and apply execution. + +To get started, there are two different options when using GitLab managed Terraform State. + +- Use a local machine +- Use GitLab CI + +## Get Started using local development + +If you are planning to only run `terraform plan` and `terraform apply` commands from your local machine, this is a simple way to get started. + +First, create your project on your GitLab instance. + +Next, define the Terraform backend in your Terraform project to be: + +```hcl +terraform { + backend "http" { + } +} +``` + +Finally, you need to run `terraform init` on your local machine and pass in the following options. The below example is using GitLab.com: + +```bash +terraform init \ + -backend-config="address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>" \ + -backend-config="lock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ + -backend-config="unlock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ + -backend-config="username=<YOUR-USERNAME>" \ + -backend-config="password=<YOUR-ACCESS-TOKEN>" \ + -backend-config="lock_method=POST" \ + -backend-config="unlock_method=DELETE" \ + -backend-config="retry_wait_min=5" +``` + +This will initialize your Terraform state and store that state within your GitLab project. + +NOTE: YOUR-PROJECT-ID and YOUR-PROJECT-NAME can be accessed from the project main page. + +## Get Started using a GitLab CI + +Another route is to leverage GitLab CI to run your `terraform plan` and `terraform apply` commands. + +### Configure the CI variables + +To use the Terraform backend, [first create a Personal Access Token](../profile/personal_access_tokens.md) with the `api` scope. Keep in mind that the Terraform backend is restricted to tokens with [Maintainer access](../permissions.md) to the repository. + +To keep the Personal Access Token secure, add it as a [CI/CD environment variable](../../ci/variables/README.md). In this example we set ours to the ENV: `GITLAB_TF_PASSWORD`. + +If you are planning to use the ENV on a branch which is not protected, make sure to set the variable protection settings correctly. + +### Configure the Terraform backend + +Next we need to define the [http backend](https://www.terraform.io/docs/backends/types/http.html). In your Terraform project add the following code block in a `.tf` file such as `backend.tf` or wherever you desire to define the remote backend: + +```hcl +terraform { + backend "http" { + } +} +``` + +### Configure the CI YAML file + +Finally, configure a `.gitlab-ci.yaml`, which lives in the root of your project repository. + +In our case we are using a pre-built image: + +```yaml +image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +``` + +We then define some environment variables to make life easier. `GITLAB_TF_ADDRESS` is the URL of the GitLab instance where this pipeline runs, and `TF_ROOT` is the directory where the Terraform commands must be executed. + +```yaml +variables: + GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} + TF_ROOT: ${CI_PROJECT_DIR}/environments/cloudflare/production + +cache: + paths: + - .terraform +``` + +In a `before_script`, pass a `terraform init` call containing configuration parameters. +These parameters correspond to variables required by the +[http backend](https://www.terraform.io/docs/backends/types/http.html): + +```yaml +before_script: + - cd ${TF_ROOT} + - terraform --version + - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=${GITLAB_USER_LOGIN}" -backend-config="password=${GITLAB_TF_PASSWORD}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" + +stages: + - validate + - build + - test + - deploy + +validate: + stage: validate + script: + - terraform validate + +plan: + stage: build + script: + - terraform plan + - terraform show + +apply: + stage: deploy + environment: + name: production + script: + - terraform apply + dependencies: + - plan + when: manual + only: + - master +``` + +### Push to GitLab + +Pushing your project to GitLab triggers a CI job pipeline, which runs the `terraform init`, `terraform validate`, and `terraform plan` commands automatically. + +The output from the above `terraform` commands should be viewable in the job logs. + +## Example project + +See [this reference project](https://gitlab.com/nicholasklick/gitlab-terraform-aws) using GitLab and Terraform to deploy a basic AWS EC2 within a custom VPC. + +## Output Terraform Plan information into a merge request + +Using the [GitLab Terraform Report Artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform), +you can expose details from `terraform plan` runs directly into a merge request widget, +enabling you to see statistics about the resources that Terraform will create, +modify, or destroy. + +Let's explore how to configure a GitLab Terraform Report Artifact: + +1. First, for simplicity, let's define a few reusable variables to allow us to + refer to these files multiple times: + + ```yaml + variables: + PLAN: plan.tfplan + PLAN_JSON: tfplan.json + ``` + +1. Next we need to install `jq`, a [lightweight and flexible command-line JSON processor](https://stedolan.github.io/jq/). We will also create an alias for a specific `jq` command that parses out the extact information we want to extract from the `terraform plan` output: + +```yaml +before_script: + - apk --no-cache add jq + - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" +``` + +1. Finally, we define a `script` that runs `terraform plan` and also a `terraform show` which pipes the output and converts the relevant bits into a store variable `PLAN_JSON`. This json is then leveraged to create a [GitLab Terraform Report Artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform). + +The terraform report obtains a Terraform tfplan.json file. The collected Terraform plan report will be uploaded to GitLab as an artifact and will be automatically shown in merge requests. + +```yaml +plan: + stage: build + script: + - terraform plan -out=$PLAN + - terraform show --json $PLAN | convert_report > $PLAN_JSON + artifacts: + name: plan + paths: + - $PLAN + reports: + terraform: $PLAN_JSON +``` + +A full `.gitlab-ci.yaml` file could look like this: + +```yaml +image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +# Default output file for Terraform plan +variables: + GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} + PLAN: plan.tfplan + PLAN_JSON: tfplan.json + TF_ROOT: ${CI_PROJECT_DIR} + +cache: + paths: + - .terraform + +before_script: + - apk --no-cache add jq + - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" + - cd ${TF_ROOT} + - terraform --version + - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=${GITLAB_USER_LOGIN}" -backend-config="password=${GITLAB_TF_PASSWORD}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" + +stages: + - validate + - build + - deploy + +validate: + stage: validate + script: + - terraform validate + +plan: + stage: build + script: + - terraform plan -out=$PLAN + - terraform show --json $PLAN | convert_report > $PLAN_JSON + artifacts: + name: plan + paths: + - ${TF_ROOT}/plan.tfplan + reports: + terraform: ${TF_ROOT}/tfplan.json + +# Separate apply job for manual launching Terraform as it can be destructive +# action. +apply: + stage: deploy + environment: + name: production + script: + - terraform apply -input=false $PLAN + dependencies: + - plan + when: manual + only: + - master + +``` + +1. Running the pipeline displays the widget in the merge request, like this: + +  + +1. Clicking the **View Full Log** button in the widget takes you directly to the + plan output present in the pipeline logs: + +  + +### Example `.gitlab-ci.yaml` file + +```yaml +image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +# Default output file for Terraform plan +variables: + GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} + PLAN: plan.tfplan + PLAN_JSON: tfplan.json + TF_ROOT: ${CI_PROJECT_DIR} + +cache: + paths: + - .terraform + +before_script: + - apk --no-cache add jq + - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" + - cd ${TF_ROOT} + - terraform --version + - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=${GITLAB_USER_LOGIN}" -backend-config="password=${GITLAB_TF_PASSWORD}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" + +stages: + - validate + - build + - deploy + +validate: + stage: validate + script: + - terraform validate + +plan: + stage: build + script: + - terraform plan -out=$PLAN + - terraform show --json $PLAN | convert_report > $PLAN_JSON + artifacts: + name: plan + paths: + - ${TF_ROOT}/plan.tfplan + reports: + terraform: ${TF_ROOT}/tfplan.json + +# Separate apply job for manual launching Terraform as it can be destructive +# action. +apply: + stage: deploy + environment: + name: production + script: + - terraform apply -input=false $PLAN + dependencies: + - plan + when: manual + only: + - master + +``` diff --git a/doc/user/instance/clusters/index.md b/doc/user/instance/clusters/index.md index 7b5ba14a5ae..495bfdeed6e 100644 --- a/doc/user/instance/clusters/index.md +++ b/doc/user/instance/clusters/index.md @@ -12,7 +12,7 @@ projects. ## Cluster precedence -GitLab will try [to match](../../../ci/environments.md#scoping-environments-with-specs) clusters in +GitLab will try [to match](../../../ci/environments/index.md#scoping-environments-with-specs) clusters in the following order: - Project-level clusters. @@ -20,11 +20,11 @@ the following order: - Instance-level clusters. To be selected, the cluster must be enabled and -match the [environment selector](../../../ci/environments.md#scoping-environments-with-specs). +match the [environment selector](../../../ci/environments/index.md#scoping-environments-with-specs). ## Cluster environments **(PREMIUM)** -For a consolidated view of which CI [environments](../../../ci/environments.md) +For a consolidated view of which CI [environments](../../../ci/environments/index.md) are deployed to the Kubernetes cluster, see the documentation for [cluster environments](../../clusters/environments.md). diff --git a/doc/user/instance_statistics/user_cohorts.md b/doc/user/instance_statistics/user_cohorts.md index 45140d5e852..b3ef99473e4 100644 --- a/doc/user/instance_statistics/user_cohorts.md +++ b/doc/user/instance_statistics/user_cohorts.md @@ -26,3 +26,4 @@ How do we measure the activity of users? GitLab considers a user active if: - The user has Git activity (whether push or pull). - The user visits pages related to Dashboards, Projects, Issues, and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/54947) in GitLab 11.8). - The user uses the API +- The user uses the GraphQL API diff --git a/doc/user/markdown.md b/doc/user/markdown.md index a1f83a47015..102eb1f8f53 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -105,7 +105,7 @@ changing how standard Markdown is used: | Standard Markdown | Extended Markdown in GitLab | | ------------------------------------- | ------------------------- | -| [blockquotes](#blockquotes) | [multiline blockquotes](#multiline-blockquote) | +| [blockquotes](#blockquotes) | [multi-line blockquotes](#multiline-blockquote) | | [code blocks](#code-spans-and-blocks) | [colored code and syntax highlighting](#colored-code-and-syntax-highlighting) | | [emphasis](#emphasis) | [multiple underscores in words](#multiple-underscores-in-words-and-mid-word-emphasis) | [headers](#headers) | [linkable Header IDs](#header-ids-and-links) | @@ -228,7 +228,7 @@ To make PlantUML available in GitLab, a GitLab administrator needs to enable it > If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#emoji). -```md +```markdown Sometimes you want to :monkey: around a bit and add some :star2: to your :speech_balloon:. Well we have a gift for you: :zap: You can use emoji anywhere GFM is supported. :v: @@ -353,7 +353,7 @@ However, the wrapping tags can't be mixed: - [- deletion -} ``` -If your diff includes words in `` `code` `` font, make sure to escape each bactick `` ` `` with a +If your diff includes words in `` `code` `` font, make sure to escape each backtick `` ` `` with a backslash `\`, otherwise the diff highlight won't render correctly: ```markdown @@ -396,8 +396,8 @@ a^2+b^2=c^2 _Be advised that KaTeX only supports a [subset](https://katex.org/docs/supported.html) of LaTeX._ -NOTE: **Note:** This also works for the asciidoctor `:stem: latexmath`. For details see -the [asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). +NOTE: **Note:** This also works for the Asciidoctor `:stem: latexmath`. For details see +the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). ### Special GitLab references @@ -608,7 +608,7 @@ Quote break. > If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#multiline-blockquote). -GFM extends the standard Markdown standard by also supporting multiline blockquotes +GFM extends the standard Markdown standard by also supporting multi-line blockquotes fenced by `>>>`: ```markdown @@ -804,7 +804,7 @@ but_emphasis is_desired _here_ If you wish to emphasize only a part of a word, it can still be done with asterisks: -```md +```markdown perform*complicated*task do*this*and*do*that*and*another thing @@ -828,23 +828,31 @@ Reference tags can use letters and other characters. Avoid using lowercase `w` o (`_`) in footnote tag names until [this bug](https://gitlab.com/gitlab-org/gitlab/issues/24423) is resolved. -```markdown -A footnote reference tag looks like this: [^1] +<!-- +Do not edit the following codeblock. It uses HTML to skip the Vale ReferenceLinks test. +--> + +<pre class="highlight"><code>A footnote reference tag looks like this: [^1] This reference tag is a mix of letters and numbers. [^footnote-42] -[^1]: This is the text inside a footnote. +[^1]: This is the text inside a footnote. -[^footnote-42]: This is another footnote. -``` +[^footnote-42]: This is another footnote. +</code></pre> A footnote reference tag looks like this:[^1] This reference tag is a mix of letters and numbers.[^footnote-42] -[^1]: This is the text inside a footnote. +<!-- +Do not delete the single space before the [^1] and [^footnotes] references below. +These are used to force the Vale ReferenceLinks check to skip these examples. +--> + + [^1]: This is the text inside a footnote. -[^footnote-42]: This is another footnote. + [^footnote-42]: This is another footnote. ### Headers @@ -928,8 +936,11 @@ ___ Examples: -```markdown -Inline-style (hover to see title text): +<!-- +Do not edit the following codeblock. It uses HTML to skip the Vale ReferenceLinks test. +--> + +<pre class="highlight"><code>Inline-style (hover to see title text):  @@ -937,12 +948,12 @@ Reference-style (hover to see title text): ![alt text1][logo] -[logo]: img/markdown_logo.png "Title Text" -``` +[logo]: img/markdown_logo.png "Title Text" +</code></pre> <!-- -DO NOT change the name of markdown_logo.png. This is used for a test -in spec/controllers/help_controller_spec.rb. +DO NOT change the name of markdown_logo.png. This is used for a test in +spec/controllers/help_controller_spec.rb. --> Inline-style (hover to see title text): @@ -951,9 +962,12 @@ Inline-style (hover to see title text): Reference-style (hover to see title text): -![alt text][logo] +<!-- +The example below uses an in-line link to pass the Vale ReferenceLinks test. +Do not change to a reference style link. +--> -[logo]: img/markdown_logo.png "Title Text" + #### Videos @@ -962,7 +976,7 @@ Reference-style (hover to see title text): Image tags that link to files with a video extension are automatically converted to a video player. The valid video extensions are `.mp4`, `.m4v`, `.mov`, `.webm`, and `.ogv`: -```md +```markdown Here's a sample video:  @@ -979,7 +993,7 @@ Here's a sample video: Similar to videos, link tags for files with an audio extension are automatically converted to an audio player. The valid audio extensions are `.mp3`, `.oga`, `.ogg`, `.spx`, and `.wav`: -```md +```markdown Here's a sample audio clip:  @@ -1036,7 +1050,10 @@ are separated into their own lines: </dl> ``` -<!-- Note: The example below uses HTML to force correct rendering on docs.gitlab.com, Markdown will be fine in GitLab --> +<!-- +Note: The example below uses HTML to force correct rendering on docs.gitlab.com, +Markdown will be fine in GitLab. +--> <dl> <dt>Markdown in HTML</dt> @@ -1102,7 +1119,10 @@ PASTE LOGS HERE </details> ```` -<!-- Note: The example below uses HTML to force correct rendering on docs.gitlab.com, Markdown will be fine in GitLab --> +<!-- +The example below uses HTML to force correct rendering on docs.gitlab.com, Markdown +will work correctly in GitLab. +--> <details> <summary>Click this to collapse/fold.</summary> @@ -1167,8 +1187,11 @@ A new line due to the previous backslash. There are two ways to create links, inline-style and reference-style: -```markdown -- This is an [inline-style link](https://www.google.com) +<!-- +Do not edit the following codeblock. It uses HTML to skip the Vale ReferenceLinks test. +--> + +<pre class="highlight"><code>- This is an [inline-style link](https://www.google.com) - This is a [link to a repository file in the same directory](index.md) - This is a [relative link to a readme one directory higher](../README.md) - This is a [link that also has title text](https://www.google.com "This link takes you to Google!") @@ -1186,14 +1209,14 @@ Using references: Some text to show that the reference links can follow later. -[arbitrary case-insensitive reference text]: https://www.mozilla.org/en-US/ -[1]: https://slashdot.org -[link text itself]: https://www.reddit.com -``` +[arbitrary case-insensitive reference text]: https://www.mozilla.org/en-US/ +[1]: https://slashdot.org +[link text itself]: https://www.reddit.com +</code></pre> - This is an [inline-style link](https://www.google.com) - This is a [link to a repository file in the same directory](index.md) -- This is a [relative link to a readme one directory higher](../README.md) +- This is a [relative link to a README one directory higher](../README.md) - This is a [link that also has title text](https://www.google.com "This link takes you to Google!") Using header ID anchors: @@ -1203,15 +1226,16 @@ Using header ID anchors: Using references: -- This is a [reference-style link, see below][Arbitrary case-insensitive reference text] -- You can [use numbers for reference-style link definitions, see below][1] -- Or leave it empty and use the [link text itself][], see below. +<!-- +The example below uses in-line links to pass the Vale ReferenceLinks test. +Do not change to reference style links. +--> -Some text to show that the reference links can follow later. +- This is a [reference-style link, see below](https://www.mozilla.org/en-US/) +- You can [use numbers for reference-style link definitions, see below](https://slashdot.org) +- Or leave it empty and use the [link text itself](https://www.reddit.com), see below. -[arbitrary case-insensitive reference text]: https://www.mozilla.org/en-US/ -[1]: https://slashdot.org -[link text itself]: https://www.reddit.com +Some text to show that the reference links can follow later. NOTE: **Note:** Relative links do not allow the referencing of project files in a wiki page, or a wiki page in a project file. The reason for this is that a wiki is always @@ -1220,7 +1244,7 @@ will point the link to `wikis/style` only when the link is inside of a wiki Mark #### URL auto-linking -GFM will autolink almost any URL you put into your text: +GFM will auto-link almost any URL you put into your text: ```markdown - https://www.google.com @@ -1251,7 +1275,7 @@ number, and count up from there. Examples: -```md +```markdown 1. First ordered list item 2. Another item - Unordered sub-list. @@ -1261,8 +1285,11 @@ Examples: 4. And another item. ``` -<!-- The "2." and "4." in the example above are changed to "1." below, to match the style standards on docs.gitlab.com --> -<!-- See https://docs.gitlab.com/ee/development/documentation/styleguide.html#lists --> +<!-- +The "2." and "4." in the example above are changed to "1." below, to match the style +standards on docs.gitlab.com. +See https://docs.gitlab.com/ee/development/documentation/styleguide.html#lists +--> 1. First ordered list item 1. Another item @@ -1275,7 +1302,7 @@ Examples: For an unordered list, add a `-`, `*` or `+`, followed by a space, at the start of each line for unordered lists, but you should not use a mix of them. -```md +```markdown Unordered lists can: - use @@ -1292,8 +1319,11 @@ They can even: + pluses ``` -<!-- The "*" and "+" in the example above are changed to "-" below, to match the style standards on docs.gitlab.com --> -<!-- See https://docs.gitlab.com/ee/development/documentation/styleguide.html#lists --> +<!-- +The "*" and "+" in the example above are changed to "-" below, to match the style +standards on docs.gitlab.com. +See https://docs.gitlab.com/ee/development/documentation/styleguide.html#lists +--> Unordered lists can: diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index ffbd8a848b5..a56a67d4959 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -20,7 +20,7 @@ by default. To enable it for existing projects, or if you want to disable it: 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. -You should then be able to see the **Packages** section on the left sidebar. +You should then be able to see the **Packages & Registries** section on the left sidebar. ## Getting started @@ -108,14 +108,19 @@ conan search Hello* --all --remote=gitlab ## Authenticating to the GitLab Conan Repository -You will need to generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. +You will need a personal access token or deploy token. + +For repository authentication: + +- You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api`. +- You can generate a [deploy token](./../../project/deploy_tokens/index.md) with the scope set to `read_package_registry`, `write_package_registry`, or both. ### Adding a Conan user to the GitLab remote Once you have a personal access token and have [set your Conan remote](#adding-the-gitlab-package-registry-as-a-conan-remote), you can associate the token with the remote so you do not have to explicitly add them to each Conan command you run: ```shell -conan user <gitlab-username> -r gitlab -p <personal_access_token> +conan user <gitlab_username or deploy_token_username> -r gitlab -p <personal_access_token or deploy_token> ``` Note: **Note** @@ -130,7 +135,7 @@ Alternatively, you could explicitly include your credentials in any given comman For example: ```shell -CONAN_LOGIN_USERNAME=<gitlab-username> CONAN_PASSWORD=<personal_access_token> conan upload Hello/0.1@my-group+my-project/beta --all --remote=gitlab +CONAN_LOGIN_USERNAME=<gitlab_username or deploy_token_username> CONAN_PASSWORD=<personal_access_token or deploy_token> conan upload Hello/0.1@my-group+my-project/beta --all --remote=gitlab ``` ### Setting a default remote to your project (optional) @@ -148,7 +153,7 @@ This functionality is best suited for when you want to consume or install packag The rest of the example commands in this documentation assume that you have added a Conan user with your credentials to the `gitlab` remote and will not include the explicit credentials or remote option, but be aware that any of the commands could be run without having added a user or default remote: ```shell -`CONAN_LOGIN_USERNAME=<gitlab-username> CONAN_PASSWORD=<personal_access_token> <conan command> --remote=gitlab +`CONAN_LOGIN_USERNAME=<gitlab_username or deploy_token_username> CONAN_PASSWORD=<personal_access_token or deploy_token> <conan command> --remote=gitlab ``` ## Uploading a package @@ -274,7 +279,7 @@ To work with Conan commands within [GitLab CI/CD](./../../../ci/README.md), you It is easiest to provide the `CONAN_LOGIN_USERNAME` and `CONAN_PASSWORD` with each Conan command in your `.gitlab-ci.yml` file: -```yml +```yaml image: conanio/gcc7 create_package: diff --git a/doc/user/packages/container_registry/img/container_registry_group_repositories_v12_10.png b/doc/user/packages/container_registry/img/container_registry_group_repositories_v12_10.png Binary files differdeleted file mode 100644 index e2b606d024f..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_group_repositories_v12_10.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_0.png b/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_0.png Binary files differnew file mode 100644 index 00000000000..14119abd56a --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_0.png diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_v12_10.png b/doc/user/packages/container_registry/img/container_registry_repositories_v12_10.png Binary files differdeleted file mode 100644 index 9e113be0a26..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repositories_v12_10.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_v13_0.png b/doc/user/packages/container_registry/img/container_registry_repositories_v13_0.png Binary files differnew file mode 100644 index 00000000000..7286007b953 --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_repositories_v13_0.png diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v12_10.png b/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v12_10.png Binary files differdeleted file mode 100644 index e94aab58a1d..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v12_10.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_0.png b/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_0.png Binary files differnew file mode 100644 index 00000000000..f7c3aafcc8e --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_0.png diff --git a/doc/user/packages/container_registry/img/container_registry_repository_details_v12.10.png b/doc/user/packages/container_registry/img/container_registry_repository_details_v12.10.png Binary files differdeleted file mode 100644 index b911ffea935..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repository_details_v12.10.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_repository_details_v13.0.png b/doc/user/packages/container_registry/img/container_registry_repository_details_v13.0.png Binary files differnew file mode 100644 index 00000000000..088e80221de --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_repository_details_v13.0.png diff --git a/doc/user/packages/container_registry/img/container_registry_tags_v12_10.png b/doc/user/packages/container_registry/img/container_registry_tags_v12_10.png Binary files differdeleted file mode 100644 index 40abc7eda9b..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_tags_v12_10.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/expiration-policy-app.png b/doc/user/packages/container_registry/img/expiration-policy-app.png Binary files differdeleted file mode 100644 index e2d3d668e38..00000000000 --- a/doc/user/packages/container_registry/img/expiration-policy-app.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/expiration_policy_app_v13_0.png b/doc/user/packages/container_registry/img/expiration_policy_app_v13_0.png Binary files differnew file mode 100644 index 00000000000..93c9e00a8f5 --- /dev/null +++ b/doc/user/packages/container_registry/img/expiration_policy_app_v13_0.png diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index 5505a4503ca..5e642e1e21c 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -8,6 +8,7 @@ > login to GitLab's Container Registry. > - Multiple level image names support was added in GitLab 9.1. > - The group level Container Registry was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23315) in GitLab 12.10. +> - Searching by image repository name was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31322) in GitLab 13.0. NOTE: **Note:** This document is the user guide. To learn how to enable GitLab Container @@ -19,14 +20,14 @@ have its own space to store its Docker images. You can read more about Docker Registry at <https://docs.docker.com/registry/introduction/>. - + ## Enable the Container Registry for your project CAUTION: **Warning:** The Container Registry follows the visibility settings of the project. If the project is public, so is the Container Registry. -If you cannot find the **Packages > Container Registry** entry under your +If you cannot find the **Packages & Registries > Container Registry** entry under your project's sidebar, it is not enabled in your GitLab instance. Ask your administrator to enable GitLab Container Registry following the [administration documentation](../../../administration/packages/container_registry.md). @@ -44,7 +45,7 @@ project: projects this might be enabled by default. For existing projects (prior GitLab 8.8), you will have to explicitly enable it. 1. Press **Save changes** for the changes to take effect. You should now be able - to see the **Packages > Container Registry** link in the sidebar. + to see the **Packages & Registries > Container Registry** link in the sidebar. ## Control Container Registry from within GitLab @@ -53,13 +54,14 @@ for both projects and groups. ### Control Container Registry for your project -Navigate to your project's **{package}** **Packages > Container Registry**. +Navigate to your project's **{package}** **Packages & Registries > Container Registry**. - + This view will: - Show all the image repositories that belong to the project. +- Allow you to filter image repositories by their name. - Allow you to [delete](#delete-images-from-within-gitlab) one or more image repository. - Allow you to navigate to the image repository details page. - Show a **Quick start** dropdown with the most common commands to log in, build and push @@ -67,9 +69,9 @@ This view will: ### Control Container Registry for your group -Navigate to your groups's **{package}** **Packages > Container Registry**. +Navigate to your groups's **{package}** **Packages & Registries > Container Registry**. - + This view will: @@ -81,7 +83,7 @@ This view will: Clicking on the name of any image repository will navigate to the details. - + NOTE: **Note:** The following page has the same functionalities both in the **Group level container registry** @@ -108,7 +110,7 @@ For more information on running Docker containers, visit the ## Authenticating to the GitLab Container Registry -If you visit the **Packages > Container Registry** link under your project's +If you visit the **Packages & Registries > Container Registry** link under your project's menu, you can see the explicit instructions to login to the Container Registry using your GitLab credentials. @@ -135,7 +137,7 @@ The minimum scope needed for both of them is `read_registry`. Example of using a token: -```sh +```shell docker login registry.example.com -u <username> -p <token> ``` @@ -152,14 +154,14 @@ docker push registry.example.com/group/project/image Your image will be named after the following scheme: -```text +```plaintext <registry URL>/<namespace>/<project>/<image> ``` GitLab supports up to three levels of image repository names. The following examples of image tags are valid: -```text +```plaintext registry.example.com/group/project:some-tag registry.example.com/group/project/image:latest registry.example.com/group/project/my/image:rc1 @@ -204,7 +206,7 @@ Available for all projects, though more suitable for public ones: your Docker images and has read/write access to the Registry. This is ephemeral, so it's only valid for one job. You can use the following example as-is: - ```sh + ```shell docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY ``` @@ -219,7 +221,7 @@ For private and internal projects: Replace the `<username>` and `<access_token>` in the following example: - ```sh + ```shell docker login -u <username> -p <access_token> $CI_REGISTRY ``` @@ -229,7 +231,7 @@ For private and internal projects: Once created, you can use the special environment variables, and GitLab CI/CD will fill them in for you. You can use the following example as-is: - ```sh + ```shell docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY ``` @@ -389,7 +391,7 @@ the deleted images. To delete images from within GitLab: -1. Navigate to your project's or group's **{package}** **Packages > Container Registry**. +1. Navigate to your project's or group's **{package}** **Packages & Registries > Container Registry**. 1. From the **Container Registry** page, you can select what you want to delete, by either: @@ -401,7 +403,7 @@ To delete images from within GitLab: 1. In the dialog box, click **Remove tag**. -  +  ### Delete images using the API @@ -490,7 +492,7 @@ older tags and images are regularly removed from the Container Registry. NOTE: **Note:** Expiration policies for projects created before GitLab 12.8 may be enabled by an admin in the [CI/CD Package Registry settings](./../../admin_area/settings/index.md#cicd). -Note the inherant [risks involved](./index.md#use-with-external-container-registries). +Note the inherent [risks involved](./index.md#use-with-external-container-registries). It is possible to create a per-project expiration policy, so that you can make sure that older tags and images are regularly removed from the Container Registry. @@ -503,23 +505,25 @@ then goes through a process of excluding tags from it until only the ones to be 1. Evaluates the `name_regex`, excluding non-matching names from the list. 1. Excludes any tags that do not have a manifest (not part of the options). 1. Orders the remaining tags by `created_date`. -1. Excludes from the list the N tags based on the `keep_n` value (Expiration latest). +1. Excludes from the list the N tags based on the `keep_n` value (Number of tags to retain). 1. Excludes from the list the tags older than the `older_than` value (Expiration interval). +1. Excludes from the list any tags matching the `name_regex_keep` value (Images to preserve). 1. Finally, the remaining tags in the list are deleted from the Container Registry. ### Managing project expiration policy through the UI To manage project expiration policy, navigate to **{settings}** **Settings > CI/CD > Container Registry tag expiration policy**. - + The UI allows you to configure the following: - **Expiration policy:** enable or disable the expiration policy. - **Expiration interval:** how long tags are exempt from being deleted. - **Expiration schedule:** how often the cron job checking the tags should run. -- **Expiration latest:** how many tags to _always_ keep for each image. +- **Number of tags to retain:** how many tags to _always_ keep for each image. - **Docker tags with names matching this regex pattern will expire:** the regex used to determine what tags should be expired. To qualify all tags for expiration, use the default value of `.*`. +- **Docker tags with names matching this regex pattern will be preserved:** the regex used to determine what tags should be preserved. To preserve all tags, use the default value of `.*`. ### Managing project expiration policy through the API @@ -527,16 +531,10 @@ You can set, update, and disable the expiration policies using the GitLab API. Examples: -- Select all tags, keep at least 1 tag per image, expire any tag older than 14 days, run once a month, and the policy is enabled: +- Select all tags, keep at least 1 tag per image, expire any tag older than 14 days, run once a month, preserve any images with the name `master` and the policy is enabled: ```shell - curl --request PUT --header 'Content-Type: application/json;charset=UTF-8' --header "PRIVATE-TOKEN: <your_access_token>" --data-binary '{"container_expiration_policy_attributes":{"cadence":"1month","enabled":true,"keep_n":1,"older_than":"14d","name_regex":".*"}}' 'https://gitlab.example.com/api/v4/projects/2' - ``` - -- Select only tags with a name that contains `stable`, keep at least 50 tag per image, expire any tag older than 7 days, run every day, and the policy is enabled: - - ```shell - curl --request PUT --header 'Content-Type: application/json;charset=UTF-8' --header "PRIVATE-TOKEN: <your_access_token>" --data-binary '{"container_expiration_policy_attributes":{"cadence":"1day","enabled":true,"keep_n":50"older_than":"7d","name_regex":"*stable"}}' 'https://gitlab.example.com/api/v4/projects/2' + curl --request PUT --header 'Content-Type: application/json;charset=UTF-8' --header "PRIVATE-TOKEN: <your_access_token>" --data-binary '{"container_expiration_policy_attributes":{"cadence":"1month","enabled":true,"keep_n":1,"older_than":"14d","name_regex":"","name_regex_delete":".*","name_regex_keep":".*-master"}}' 'https://gitlab.example.com/api/v4/projects/2' ``` See the API documentation for further details: [Edit project](../../../api/projects.md#edit-project). @@ -544,7 +542,7 @@ See the API documentation for further details: [Edit project](../../../api/proje ### Use with external container registries When using an [external container registry](./../../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint), -running an experation policy on a project may have some performance risks. If a project is going to run +running an expiration policy on a project may have some performance risks. If a project is going to run a policy that will remove large quantities of tags (in the thousands), the GitLab background jobs that run the policy may get backed up or fail completely. It is recommended you only enable container expiration policies for projects that were created before GitLab 12.8 if you are confident the amount of tags @@ -552,10 +550,12 @@ being cleaned up will be minimal. ## Limitations -Moving or renaming existing Container Registry repositories is not supported +- Moving or renaming existing Container Registry repositories is not supported once you have pushed images, because the images are signed, and the signature includes the repository name. To move or rename a repository with a Container Registry, you will have to delete all existing images. +- Prior to GitLab 12.10, any tags that use the same image ID as the `latest` tag +will not be deleted by the expiration policy. ## Troubleshooting the GitLab Container Registry @@ -577,3 +577,47 @@ Troubleshooting the GitLab Container Registry, most of the times, requires administration access to the GitLab server. [Read how to troubleshoot the Container Registry](../../../administration/packages/container_registry.md#troubleshooting). + +### Unable to change path or transfer a project + +If you try to change a project's path or transfer a project to a new namespace, +you may receive one of the following errors: + +- "Project cannot be transferred, because tags are present in its container registry." +- "Namespace cannot be moved because at least one project has tags in container registry." + +This issue occurs when the project has images in the Container Registry. +You must delete or move these images before you can change the path or transfer +the project. + +The following procedure uses these sample project names: + +- For the current project: `example.gitlab.com/org/build/sample_project/cr:v2.9.1` +- For the new project: `example.gitlab.com/new_org/build/new_sample_project/cr:v2.9.1` + +Use your own URLs to complete the following steps: + +1. Download the Docker images on your computer: + + ```shell + docker login example.gitlab.com + docker pull example.gitlab.com/org/build/sample_project/cr:v2.9.1 + ``` + +1. Rename the images to match the new project name: + + ```shell + docker tag example.gitlab.com/org/build/sample_project/cr:v2.9.1 example.gitlab.com/new_org/build/new_sample_project/cr:v2.9.1 + ``` + +1. Delete the images in both projects by using the [UI](#delete-images) or [API](../../../api/packages.md#delete-a-project-package). + There may be a delay while the images are queued and deleted. +1. Change the path or transfer the project by going to **Settings > General** + and expanding **Advanced**. +1. Restore the images: + + ```shell + docker push example.gitlab.com/new_org/build/new_sample_project/cr:v2.9.1 + ``` + +Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/18383) for details. diff --git a/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png b/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png Binary files differindex 0b94efdd83e..e550d296d5a 100644 --- a/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png +++ b/doc/user/packages/dependency_proxy/img/group_dependency_proxy.png diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index cfdcd9821fb..be9710053dd 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -12,7 +12,7 @@ receiving a request and returning the upstream image from a registry, acting as a pull-through cache. The dependency proxy is available in the group level. To access it, navigate to -a group's **Packages > Dependency Proxy**. +a group's **Packages & Registries > Dependency Proxy**.  @@ -33,7 +33,7 @@ The following dependency proxies are supported. With the Docker dependency proxy, you can use GitLab as a source for a Docker image. To get a Docker image into the dependency proxy: -1. Find the proxy URL on your group's page under **Packages > Dependency Proxy**, +1. Find the proxy URL on your group's page under **Packages & Registries > Dependency Proxy**, for example `gitlab.com/groupname/dependency_proxy/containers`. 1. Trigger GitLab to pull the Docker image you want (e.g., `alpine:latest` or `linuxserver/nextcloud:latest`) and store it in the proxy storage by using diff --git a/doc/user/packages/img/group_packages_list_v12_10.png b/doc/user/packages/img/group_packages_list_v12_10.png Binary files differdeleted file mode 100644 index ba9f2892961..00000000000 --- a/doc/user/packages/img/group_packages_list_v12_10.png +++ /dev/null diff --git a/doc/user/packages/img/group_packages_list_v13_0.png b/doc/user/packages/img/group_packages_list_v13_0.png Binary files differnew file mode 100644 index 00000000000..8cf3fd1a131 --- /dev/null +++ b/doc/user/packages/img/group_packages_list_v13_0.png diff --git a/doc/user/packages/img/package_detail_v12_10.png b/doc/user/packages/img/package_detail_v12_10.png Binary files differdeleted file mode 100644 index b2cd8e31955..00000000000 --- a/doc/user/packages/img/package_detail_v12_10.png +++ /dev/null diff --git a/doc/user/packages/img/package_detail_v13_0.png b/doc/user/packages/img/package_detail_v13_0.png Binary files differnew file mode 100644 index 00000000000..dcfbc0a4787 --- /dev/null +++ b/doc/user/packages/img/package_detail_v13_0.png diff --git a/doc/user/packages/img/project_packages_list_v12_10.png b/doc/user/packages/img/project_packages_list_v12_10.png Binary files differdeleted file mode 100644 index 2dfb92fa796..00000000000 --- a/doc/user/packages/img/project_packages_list_v12_10.png +++ /dev/null diff --git a/doc/user/packages/img/project_packages_list_v13_0.png b/doc/user/packages/img/project_packages_list_v13_0.png Binary files differnew file mode 100644 index 00000000000..468a6fe6467 --- /dev/null +++ b/doc/user/packages/img/project_packages_list_v13_0.png diff --git a/doc/user/packages/index.md b/doc/user/packages/index.md index 8e98dd70346..132a64d99a3 100644 --- a/doc/user/packages/index.md +++ b/doc/user/packages/index.md @@ -18,26 +18,28 @@ The Packages feature allows GitLab to act as a repository for the following: ## Enable the Package Registry for your project -If you cannot find the **{package}** **Packages > List** entry under your -project's sidebar, it is not enabled in your GitLab instance. Ask your -administrator to enable GitLab Package Registry following the administration -documentation. +If you cannot find the **{package}** **Packages & Registries > Package +Registry** entry under your project's sidebar, ensure that: -Once enabled for your GitLab instance, to enable Package Registry for your -project: +1. The GitLab Package Registry has been enabled by your administrator (following + [this documentation](../../administration/packages/index.md)); and +1. The Package Registry has been enabled for your project. + +Once an administrator has enabled the GitLab Package Registry for your GitLab +instance, to enable Package Registry for your project: 1. Go to your project's **Settings > General** page. 1. Expand the **Visibility, project features, permissions** section and enable the **Packages** feature on your project. 1. Press **Save changes** for the changes to take effect. You should now be able to -see the **Packages > List** link in the sidebar. +see the **Packages & Registries > Package Registry** link in the sidebar. ### View Packages for your project -Navigating to your project's **{package}** **Packages > List** will show a list +Navigating to your project's **{package}** **Packages & Registries > Package Registry** will show a list of all packages that have been added to your project. - + On this page, you can: @@ -51,9 +53,9 @@ On this page, you can: ### View Packages for your group You can view all packages belonging to a group by navigating to **{package}** -**Packages > List** from the group sidebar. +**Packages & Registries > Package Registry** from the group sidebar. - + On this page, you can: @@ -68,7 +70,7 @@ On this page, you can: Additional package information can be viewed by browsing to the package details page from the either the project or group list. - + On this page you can: @@ -112,7 +114,6 @@ are adding support for [PHP](https://gitlab.com/gitlab-org/gitlab/-/merge_reques | [Opkg](https://gitlab.com/gitlab-org/gitlab/issues/36894) | Optimize your work with OpenWrt using Opkg repositories. | | [P2](https://gitlab.com/gitlab-org/gitlab/issues/36895) | Host all your Eclipse plugins in your own GitLab P2 repository. | | [Puppet](https://gitlab.com/gitlab-org/gitlab/issues/36897) | Configuration management meets repository management with Puppet repositories. | -| [PyPi](https://gitlab.com/gitlab-org/gitlab/issues/10483) | Host PyPi distributions. | | [RPM](https://gitlab.com/gitlab-org/gitlab/issues/5932) | Distribute RPMs directly from GitLab. | | [RubyGems](https://gitlab.com/gitlab-org/gitlab/issues/803) | Use GitLab to host your own gems. | | [SBT](https://gitlab.com/gitlab-org/gitlab/issues/36898) | Resolve dependencies from and deploy build output to SBT repositories when running SBT builds. | diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index b4aec11d029..51e62dc871e 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -21,17 +21,19 @@ to disable it: 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. -You should then be able to see the **Packages** section on the left sidebar. +You should then be able to see the **Packages & Registries** section on the left sidebar. Next, you must configure your project to authorize with the GitLab Maven repository. -## Getting Started +## Getting Started with Maven This section will cover installing Maven and building a package. This is a quickstart to help if you're new to building Maven packages. If you're already using Maven and understand how to build your own packages, move onto the [next section](#adding-the-gitlab-package-registry-as-a-maven-remote). +Maven repositories work well with Gradle, too. Move onto [getting started with Gradle](#getting-started-with-gradle) if you want to setup a Gradle project. + ### Installing Maven The required minimum versions are: @@ -96,20 +98,129 @@ your project has been set up successfully: You should see a new directory where you ran this command matching your `DartifactId` parameter (in this case it should be `my-project`). +## Getting started with Gradle + +This section will cover installing Gradle and initializing a Java project. This is a +quickstart to help if you're new to Gradle. If you're already +using Gradle and understand how to build your own packages, move onto the +[next section](#adding-the-gitlab-package-registry-as-a-maven-remote). + +### Installing Gradle + +Installation is needed only if you want to create a new Gradle project. Follow +instructions at [gradle.org](https://gradle.org/install/) to download and install +Gradle for your local development environment. + +Verify you can use Gradle in your terminal by running: + +```shell +gradle -version +``` + +If you want to use an existing Gradle project, installation is not necessary. +Simply execute `gradlew` (on Linux) or `gradlew.bat` (on Windows) in the project +directory instead. + +You should see something imilar to the below printed in the output: + +```plaintext +------------------------------------------------------------ +Gradle 6.0.1 +------------------------------------------------------------ + +Build time: 2019-11-18 20:25:01 UTC +Revision: fad121066a68c4701acd362daf4287a7c309a0f5 + +Kotlin: 1.3.50 +Groovy: 2.5.8 +Ant: Apache Ant(TM) version 1.10.7 compiled on September 1 2019 +JVM: 11.0.5 (Oracle Corporation 11.0.5+10) +OS: Windows 10 10.0 amd64 +``` + +### Creating a project in Gradle + +Understanding how to create a full Java project in Gradle is outside the scope of this +guide, but you can follow the steps below to create a new project that can be +published to the GitLab Package Registry. + +Start by opening your terminal and creating a directory where you would like to +store the project in your environment. From inside the directory, you can run +the following Maven command to initialize a new package: + +```shell +gradle init +``` + +The output should be + +```plaintext +Select type of project to generate: + 1: basic + 2: application + 3: library + 4: Gradle plugin +Enter selection (default: basic) [1..4] +``` + +Enter `3` to create a new Library project. The output should be: + +```plaintext +Select implementation language: + 1: C++ + 2: Groovy + 3: Java + 4: Kotlin + 5: Scala + 6: Swift +``` + +Enter `3` to create a new Java Library project. The output should be: + +```plaintext +Select build script DSL: + 1: Groovy + 2: Kotlin +Enter selection (default: Groovy) [1..2] +``` + +Choose `1` to create a new Java Library project which will be described in Groovy DSL. The output should be: + +```plaintext +Select test framework: + 1: JUnit 4 + 2: TestNG + 3: Spock + 4: JUnit Jupiter +``` + +Choose `1` to initialize the project with JUnit 4 testing libraries. The output should be: + +```plaintext +Project name (default: test): +``` + +Enter a project name or hit enter to use the directory name as project name. + ## Adding the GitLab Package Registry as a Maven remote The next step is to add the GitLab Package Registry as a Maven remote. If a project is private or you want to upload Maven artifacts to GitLab, credentials will need to be provided for authorization too. Support is available -for [personal access tokens](#authenticating-with-a-personal-access-token) and -[CI job tokens](#authenticating-with-a-ci-job-token) only. -[Deploy tokens](../../project/deploy_tokens/index.md) and regular username/password +for [personal access tokens](#authenticating-with-a-personal-access-token), +[CI job tokens](#authenticating-with-a-ci-job-token), and +[deploy tokens](../../project/deploy_tokens/index.md) only. Regular username/password credentials do not work. ### Authenticating with a personal access token To authenticate with a [personal access token](../../profile/personal_access_tokens.md), -set the scope to `api` and add a corresponding section to your +set the scope to `api` when creating one, and add it to your Maven or Gradle configuration +files. + +#### Authenticating with a personal access token in Maven + +Add a corresponding section to your [`settings.xml`](https://maven.apache.org/settings.html) file: ```xml @@ -130,13 +241,43 @@ set the scope to `api` and add a corresponding section to your </settings> ``` +#### Authenticating with a personal access token in Gradle + +Create a file `~/.gradle/gradle.properties` with the following content: + +```groovy +gitLabPrivateToken=REPLACE_WITH_YOUR_PERSONAL_ACCESS_TOKEN +``` + +Add a repositories section to your +[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) +file: + +```groovy +repositories { + maven { + url "https://<gitlab-url>/api/v4/groups/<group>/-/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = 'Private-Token' + value = gitLabPrivateToken + } + authentication { + header(HttpHeaderAuthentication) + } + } +} +``` + You should now be able to upload Maven artifacts to your project. ### Authenticating with a CI job token -If you're using Maven with GitLab CI/CD, a CI job token can be used instead +If you're using GitLab CI/CD, a CI job token can be used instead of a personal access token. +#### Authenticating with a CI job token in Maven + To authenticate with a CI job token, add a corresponding section to your [`settings.xml`](https://maven.apache.org/settings.html) file: @@ -161,6 +302,81 @@ To authenticate with a CI job token, add a corresponding section to your You can read more on [how to create Maven packages using GitLab CI/CD](#creating-maven-packages-with-gitlab-cicd). +#### Authenticating with a CI job token in Gradle + +To authenticate with a CI job token, add a repositories section to your +[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) +file: + +```groovy +repositories { + maven { + url "https://<gitlab-url>/api/v4/groups/<group>/-/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = 'Job-Token' + value = '${CI_JOB_TOKEN}' + } + authentication { + header(HttpHeaderAuthentication) + } + } +} +``` + +### Authenticating with a deploy token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. + +To authenticate with a [deploy token](./../../project/deploy_tokens/index.md), +set the scope to `api` when creating one, and add it to your Maven or Gradle configuration +files. + +#### Authenticating with a deploy token in Maven + +Add a corresponding section to your +[`settings.xml`](https://maven.apache.org/settings.html) file: + +```xml +<settings> + <servers> + <server> + <id>gitlab-maven</id> + <configuration> + <httpHeaders> + <property> + <name>Deploy-Token</name> + <value>REPLACE_WITH_YOUR_DEPLOY_TOKEN</value> + </property> + </httpHeaders> + </configuration> + </server> + </servers> +</settings> +``` + +#### Authenticating with a deploy token in Gradle + +To authenticate with a deploy token, add a repositories section to your +[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) +file: + +```groovy +repositories { + maven { + url "https://<gitlab-url>/api/v4/groups/<group>/-/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = 'Deploy-Token' + value = '<deploy-token>' + } + authentication { + header(HttpHeaderAuthentication) + } + } +} +``` + ## Configuring your project to use the GitLab Maven repository URL To download and upload packages from GitLab, you need a `repository` and @@ -185,7 +401,7 @@ the `distributionManagement` section. ### Project level Maven endpoint The example below shows how the relevant `repository` section of your `pom.xml` -would look like: +would look like in Maven: ```xml <repositories> @@ -206,6 +422,17 @@ would look like: </distributionManagement> ``` +The corresponding section in Gradle would look like this: + +```groovy +repositories { + maven { + url "https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven" + name "GitLab" + } +} +``` + The `id` must be the same with what you [defined in `settings.xml`](#adding-the-gitlab-package-registry-as-a-maven-remote). @@ -223,7 +450,7 @@ project's ID can be used for uploading. ### Group level Maven endpoint -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8798) in GitLab Premium 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8798) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. If you rely on many packages, it might be inefficient to include the `repository` section with a unique URL for each package. Instead, you can use the group level endpoint for @@ -259,6 +486,17 @@ the `distributionManagement` section: </distributionManagement> ``` +For Gradle, the corresponding repositories section would look like: + +```groovy +repositories { + maven { + url "https://gitlab.com/api/v4/groups/GROUP_ID/-/packages/maven" + name "GitLab" + } +} +``` + The `id` must be the same with what you [defined in `settings.xml`](#adding-the-gitlab-package-registry-as-a-maven-remote). @@ -275,7 +513,7 @@ For retrieving artifacts, you can use either the ### Instance level Maven endpoint -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8274) in GitLab Premium 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8274) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. If you rely on many packages, it might be inefficient to include the `repository` section with a unique URL for each package. Instead, you can use the instance level endpoint for @@ -314,6 +552,17 @@ the `distributionManagement` section: </distributionManagement> ``` +The corresponding repositories section in Gradle would look like: + +```groovy +repositories { + maven { + url "https://gitlab.com/api/v4/packages/maven" + name "GitLab" + } +} +``` + The `id` must be the same with what you [defined in `settings.xml`](#adding-the-gitlab-package-registry-as-a-maven-remote). @@ -333,7 +582,9 @@ project's ID can be used for uploading. Once you have set up the [remote and authentication](#adding-the-gitlab-package-registry-as-a-maven-remote) and [configured your project](#configuring-your-project-to-use-the-gitlab-maven-repository-url), -test to upload a Maven artifact from a project of yours: +test to upload a Maven artifact from a project of yours. + +### Upload using Maven ```shell mvn deploy @@ -353,7 +604,51 @@ You should also see that the upload was uploaded to the correct registry: Uploading to gitlab-maven: https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.jar ``` -You can then navigate to your project's **Packages** page and see the uploaded +### Upload using Gradle + +Add the Gradle plugin [`maven-publish`](https://docs.gradle.org/current/userguide/publishing_maven.html) to the plugins section: + +```groovy +plugins { + id 'java' + id 'maven-publish' +} +``` + +Add a `publishing` section: + +```groovy +publishing { + publications { + library(MavenPublication) { + from components.java + } + } + repositories { + maven { + url "https://gitlab.com/api/v4/projects/<PROJECT_ID>/packages/maven" + credentials(HttpHeaderCredentials) { + name = "Private-Token" + value = gitLabPrivateToken // the variable resides in ~/.gradle/gradle.properties + } + authentication { + header(HttpHeaderAuthentication) + } + } + } +} +``` + +Replace `PROJECT_ID` with your project ID which can be found on the home page +of your project. + +Run the publish task: + +```shell +gradle publish +``` + +You can then navigate to your project's **Packages & Registries** page and see the uploaded artifacts or even delete them. ## Installing a package @@ -362,7 +657,7 @@ Installing a package from the GitLab Package Registry requires that you set up the [remote and authentication](#adding-the-gitlab-package-registry-as-a-maven-remote) as above. Once this is completed, there are two ways for installaing a package. -### Install with `mvn install` +### Install using Maven with `mvn install` Add the dependency manually to your project `pom.xml` file. To add the example created above, the XML would look like: @@ -388,7 +683,7 @@ downloaded from the GitLab Package Registry: Downloading from gitlab-maven: http://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom ``` -### Install with `mvn dependency:get` +#### Install with `mvn dependency:get` The second way to install packages is to use the Maven commands directly. Inside your project directory, run: @@ -404,6 +699,16 @@ TIP: **Tip:** Both the XML block and Maven command are readily copy and pastable from the Package details page, allowing for quick and easy installation. +### Install using Gradle + +Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to build.gradle in the dependencies section: + +```groovy +dependencies { + implementation 'com.mycompany.mydepartment:my-project:1.0-SNAPSHOT' +} +``` + ## Removing a package In the packages view of your project page, you can delete packages by clicking @@ -413,11 +718,15 @@ page. ## Creating Maven packages with GitLab CI/CD Once you have your repository configured to use the GitLab Maven Repository, -you can configure GitLab CI/CD to build new packages automatically. The example below -shows how to create a new package each time the `master` branch is updated: +you can configure GitLab CI/CD to build new packages automatically. + +### Creating Maven packages with GitLab CI/CD using Maven + +The example below shows how to create a new package each time the `master` branch +is updated: 1. Create a `ci_settings.xml` file that will serve as Maven's `settings.xml` file. - Add the server section with the same id you defined in your `pom.xml` file. + Add the server section with the same ID you defined in your `pom.xml` file. For example, in our case it's `gitlab-maven`: ```xml @@ -481,6 +790,31 @@ user's home location (in this case the user is `root` since it runs in a Docker container), and Maven will utilize the configured CI [environment variables](../../../ci/variables/README.md#predefined-environment-variables). +### Creating Maven packages with GitLab CI/CD using Gradle + +The example below shows how to create a new package each time the `master` branch +is updated: + +1. Make sure you use the Job-Token authentication as described in ["Authenticating with a CI job token in Gradle"](#authenticating-with-a-ci-job-token-in-gradle). + +1. Add a `deploy` job to your `.gitlab-ci.yml` file: + + ```yaml + deploy: + image: gradle:latest + script: + - 'gradle publish' + only: + - master + ``` + +1. Push those files to your repository. + +The next time the `deploy` job runs, it will copy `ci_settings.xml` to the +user's home location (in this case the user is `root` since it runs in a +Docker container), and Maven will use the configured CI +[environment variables](../../../ci/variables/README.md#predefined-environment-variables). + ## Troubleshooting ### Useful Maven command line options diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index e66b3d1ac63..b909646431b 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -23,7 +23,7 @@ by default. To enable it for existing projects, or if you want to disable it: 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. -You should then be able to see the **Packages** section on the left sidebar. +You should then be able to see the **Packages & Registries** section on the left sidebar. Before proceeding to authenticating with the GitLab NPM Registry, you should get familiar with the package naming convention. @@ -100,14 +100,15 @@ configure GitLab as a remote registry. If a project is private or you want to upload an NPM package to GitLab, credentials will need to be provided for authentication. [Personal access tokens](../../profile/personal_access_tokens.md) +and [deploy tokens](../../project/deploy_tokens/index.md) are preferred, but support is available for [OAuth tokens](../../../api/oauth2.md#resource-owner-password-credentials-flow). -CAUTION: **2FA is only supported with personal access tokens:** -If you have 2FA enabled, you need to use a [personal access token](../../profile/personal_access_tokens.md) with OAuth headers with the scope set to `api`. Standard OAuth tokens won't be able to authenticate to the GitLab NPM Registry. +CAUTION: **Two-factor authentication (2FA) is only supported with personal access tokens:** +If you have 2FA enabled, you need to use a [personal access token](../../profile/personal_access_tokens.md) with OAuth headers with the scope set to `api` or a [deploy token](../../project/deploy_tokens/index.md) with `read_package_registry` or `write_package_registry` scopes. Standard OAuth tokens won't be able to authenticate to the GitLab NPM Registry. -### Authenticating with a personal access token +### Authenticating with a personal access token or deploy token -To authenticate with a [personal access token](../../profile/personal_access_tokens.md), +To authenticate with a [personal access token](../../profile/personal_access_tokens.md) or [deploy token](../../project/deploy_tokens/index.md), set your NPM configuration: ```shell @@ -125,7 +126,7 @@ npm config set '//gitlab.com/api/v4/projects/<your_project_id>/packages/npm/:_au ``` Replace `<your_project_id>` with your project ID which can be found on the home page -of your project and `<your_token>` with your personal access token. +of your project and `<your_token>` with your personal access token or deploy token. If you have a self-managed GitLab installation, replace `gitlab.com` with your domain name. @@ -160,7 +161,7 @@ Then, you could run `npm publish` either locally or via GitLab CI/CD: > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9104) in GitLab Premium 12.5. -If you’re using NPM with GitLab CI/CD, a CI job token can be used instead of a personal access token. +If you’re using NPM with GitLab CI/CD, a CI job token can be used instead of a personal access token or deploy token. The token will inherit the permissions of the user that generates the pipeline. Add a corresponding section to your `.npmrc` file: @@ -195,7 +196,7 @@ you can upload an NPM package to your project: npm publish ``` -You can then navigate to your project's **Packages** page and see the uploaded +You can then navigate to your project's **Packages & Registries** page and see the uploaded packages or even delete them. If you attempt to publish a package with a name that already exists within @@ -286,11 +287,11 @@ page. ## Publishing a package with CI/CD To work with NPM commands within [GitLab CI/CD](./../../../ci/README.md), you can use -`CI_JOB_TOKEN` in place of the personal access token in your commands. +`CI_JOB_TOKEN` in place of the personal access token or deploy token in your commands. A simple example `.gitlab-ci.yml` file for publishing NPM packages: -```yml +```yaml image: node:latest stages: @@ -323,9 +324,9 @@ info Visit https://classic.yarnpkg.com/en/docs/cli/install for documentation abo ``` In this case, try adding this to your `.npmrc` file (and replace `<your_token>` -with your personal access token): +with your personal access token or deploy token): -```text +```plaintext //gitlab.com/api/v4/projects/:_authToken=<your_token> ``` @@ -363,6 +364,14 @@ You do not need a token to run `npm install` unless your project is private (the NPM_TOKEN=<your_token> npm install ``` +### `npm install` returns `npm ERR! 403 Forbidden` + +- Check that your token is not expired and has appropriate permissions. +- Check if you have attempted to publish a package with a name that already exists within a given scope. +- Ensure the scoped packages URL includes a trailing slash: + - Correct: `//gitlab.com/api/v4/packages/npm/` + - Incorrect: `//gitlab.com/api/v4/packages/npm` + ## NPM dependencies metadata > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11867) in GitLab Premium 12.6. diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index ed936b546d2..d9efb3239a8 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -61,14 +61,16 @@ by default. To enable it for existing projects, or if you want to disable it: 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. -You should then be able to see the **Packages** section on the left sidebar. +You should then be able to see the **Packages & Registries** section on the left sidebar. ## Adding the GitLab NuGet Repository as a source to NuGet You will need the following: - Your GitLab username. -- A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. +- A personal access token or deploy token. For repository authentication: + - You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api`. + - You can generate a [deploy token](./../../project/deploy_tokens/index.md) with the scope set to `read_package_registry`, `write_package_registry`, or both. - A suitable name for your source. - Your project ID which can be found on the home page of your project. @@ -83,7 +85,7 @@ You can now add a new source to NuGet with: To add the GitLab NuGet Repository as a source with `nuget`: ```shell -nuget source Add -Name <source_name> -Source "https://gitlab-instance.example.com/api/v4/projects/<your_project_id>/packages/nuget/index.json" -UserName <gitlab_username> -Password <gitlab_personal_access_token> +nuget source Add -Name <source_name> -Source "https://gitlab-instance.example.com/api/v4/projects/<your_project_id>/packages/nuget/index.json" -UserName <gitlab_username or deploy_token_username> -Password <gitlab_personal_access_token or deploy_token> ``` Where: @@ -107,8 +109,8 @@ nuget source Add -Name "GitLab" -Source "https//gitlab.example/api/v4/projects/1 - **Location**: `https://gitlab.com/api/v4/projects/<your_project_id>/packages/nuget/index.json` - Replace `<your_project_id>` with your project ID. - If you have a self-managed GitLab installation, replace `gitlab.com` with your domain name. - - **Username**: Your GitLab username - - **Password**: Your personal access token + - **Username**: Your GitLab username or deploy token username + - **Password**: Your personal access token or deploy token  @@ -131,8 +133,8 @@ To add the GitLab NuGet Repository as a source for .NET, create a file named `nu </packageSources> <packageSourceCredentials> <gitlab> - <add key="Username" value="<gitlab_username>" /> - <add key="ClearTextPassword" value="<gitlab_personal_access_token>" /> + <add key="Username" value="<gitlab_username or deploy_token_username>" /> + <add key="ClearTextPassword" value="<gitlab_personal_access_token or deploy_token>" /> </gitlab> </packageSourceCredentials> </configuration> @@ -201,7 +203,7 @@ nuget install <package_id> -OutputDirectory <output_directory> \ Where: -- `<package_id>` is the package id. +- `<package_id>` is the package ID. - `<output_directory>` is the output directory, where the package will be installed. - `<package_version>` (Optional) is the package version. - `<source_name>` (Optional) is the source name. @@ -222,5 +224,5 @@ dotnet add package <package_id> \ Where: -- `<package_id>` is the package id. +- `<package_id>` is the package ID. - `<package_version>` (Optional) is the package version. diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index 11d7b828813..979f7a3c966 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -26,10 +26,132 @@ by default. To enable it for existing projects, or if you want to disable it: 1. Find the Packages feature and enable or disable it. 1. Click on **Save changes** for the changes to take effect. -You should then be able to see the **Packages** section on the left sidebar. +You should then be able to see the **Packages & Registries** section on the left sidebar. + +## Getting started + +This section will cover creating a new example PyPi package to upload. This is a +quickstart to test out the **GitLab PyPi Registry**. If you already understand how +to build and publish your own packages, move on to the [next section](#adding-the-gitlab-pypi-repository-as-a-source). + +### Create a project + +Understanding how to create a full Python project is outside the scope of this +guide, but you can create a small package to test out the registry. Start by +creating a new directory called `MyPyPiPackage`: + +```shell +mkdir MyPyPiPackage && cd MyPyPiPackage +``` + +After creating this, create another directory inside: + +```shell +mkdir mypypipackage && cd mypypipackage +``` + +Create two new files inside this directory to set up the basic project: + +```shell +touch __init__.py +touch greet.py +``` + +Inside `greet.py`, add the following code: + +```python +def SayHello(): + print("Hello from MyPyPiPackage") + return +``` + +Inside the `__init__.py` file, add the following: + +```python +from .greet import SayHello +``` + +Now that the basics of our project is completed, we can test that the code runs. +Start the Python prompt inside your top `MyPyPiPackage` directory. Then run: + +```python +>>> from mypypipackage import SayHello +>>> SayHello() +``` + +You should see an output similar to the following: + +```plaintext +Python 3.8.2 (v3.8.2:7b3ab5921f, Feb 24 2020, 17:52:18) +[Clang 6.0 (clang-600.0.57)] on darwin +Type "help", "copyright", "credits" or "license" for more information. +>>> from mypypipackage import SayHello +>>> SayHello() +Hello from MyPyPiPackage +``` + +Once we've verified that the sample project is working as above, we can next +work on creating a package. + +### Create a package + +Inside your `MyPyPiPackage` directory, we need to create a `setup.py` file. Run +the following: + +```shell +touch setup.py +``` + +This file contains all the information about our package. For more information +about this file, see [creating setup.py](https://packaging.python.org/tutorials/packaging-projects/#creating-setup-py). +For this guide, we don't need to extensively fill out this file, simply add the +below to your `setup.py`: + +```python +import setuptools + +setuptools.setup( + name="mypypipackage", + version="0.0.1", + author="Example Author", + author_email="author@example.com", + description="A small example package", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires='>=3.6', +) +``` + +Save the file, then execute the setup like so: + +```shell +python3 setup.py sdist bdist_wheel +``` + +If successful, you should be able to see the output in a newly created `dist` +folder. Run: + +```shell +ls dist +``` + +And confirm your output matches the below: + +```plaintext +mypypipackage-0.0.1-py3-none-any.whl mypypipackage-0.0.1.tar.gz +``` + +Our package is now all set up and ready to be uploaded to the **GitLab PyPi +Package Registry**. Before we do so, we next need to set up authentication. ## Adding the GitLab PyPi Repository as a source +### Authenticating with a personal access token + You will need the following: - A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. @@ -39,12 +161,37 @@ You will need the following: Edit your `~/.pypirc` file and add the following: ```ini +[distutils] +index-servers = + gitlab + [gitlab] repository = https://gitlab.com/api/v4/projects/<project_id>/packages/pypi username = __token__ password = <your personal access token> ``` +### Authenticating with a deploy token + +You will need the following: + +- A deploy token. You can generate a [deploy token](./../../project/deploy_tokens/index.md) with the `read_package_registry` and/or `write_package_registry` scopes for repository authentication. +- A suitable name for your source. +- Your project ID which can be found on the home page of your project. + +Edit your `~/.pypirc` file and add the following: + +```ini +[distutils] +index-servers = + gitlab + +[gitlab] +repository = https://gitlab.com/api/v4/projects/<project_id>/packages/pypi +username = <deploy token username> +password = <deploy token> +``` + ## Uploading packages When uploading packages, note that: @@ -52,13 +199,33 @@ When uploading packages, note that: - The maximum allowed size is 50 Megabytes. - If you upload the same package with the same version multiple times, each consecutive upload is saved as a separate file. When installing a package, GitLab will serve the most recent file. -- When uploading packages to GitLab, they will not be displayed in the packages UI of your project - immediately. It can take up to 10 minutes to process a package. ### Upload packages with Twine -This section assumes that your project is properly built and you already [created a PyPi package with setuptools](https://packaging.python.org/tutorials/packaging-projects/). -Upload your package using the following command: +If you were following the guide above, then the `MyPyPiPackage` package should +be ready to be uploaded. Run the following command: + +```shell +python3 -m twine upload --repository gitlab dist/* +``` + +If successful, you should see the following: + +```plaintext +Uploading distributions to https://gitlab.com/api/v4/projects/<your_project_id>/packages/pypi +Uploading mypypipackage-0.0.1-py3-none-any.whl +100%|███████████████████████████████████████████████████████████████████████████████████████████| 4.58k/4.58k [00:00<00:00, 10.9kB/s] +Uploading mypypipackage-0.0.1.tar.gz +100%|███████████████████████████████████████████████████████████████████████████████████████████| 4.24k/4.24k [00:00<00:00, 11.0kB/s] +``` + +This indicates that the package was uploaded successfully. You can then navigate +to your project's **Packages & Registries** page and see the uploaded packages. + +If you did not follow the guide above, the you'll need to ensure your package +has been properly built and you [created a PyPi package with setuptools](https://packaging.python.org/tutorials/packaging-projects/). + +You can then upload your package using the following command: ```shell python -m twine upload --repository <source_name> dist/<package_file> @@ -80,5 +247,22 @@ pip install --index-url https://__token__:<personal_access_token>@gitlab.com/api Where: - `<package_name>` is the package name. -- `<personal_access_token>` is your personal access token. -- `<project_id>` is your project id number. +- `<personal_access_token>` is a personal access token with the `read_api` scope. +- `<project_id>` is the project ID. + +If you were following the guide above and want to test installing the +`MyPyPiPackage` package, you can run the following: + +```shell +pip install mypypipackage --no-deps --index-url https://__token__:<personal_access_token>@gitlab.com/api/v4/projects/<your_project_id>/packages/pypi/simple +``` + +This should result in the following: + +```plaintext +Looking in indexes: https://__token__:****@gitlab.com/api/v4/projects/<your_project_id>/packages/pypi/simple +Collecting mypypipackage + Downloading https://gitlab.com/api/v4/projects/<your_project_id>/packages/pypi/files/d53334205552a355fee8ca35a164512ef7334f33d309e60240d57073ee4386e6/mypypipackage-0.0.1-py3-none-any.whl (1.6 kB) +Installing collected packages: mypypipackage +Successfully installed mypypipackage-0.0.1 +``` diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 5a631cc59e3..e5893b291dc 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -42,7 +42,7 @@ NOTE: **Note:** In GitLab 11.0, the Master role was renamed to Maintainer. While Maintainer is the highest project-level role, some actions can only be performed by a personal namespace or group owner, -or an instance admin, who receives all permissions. +or an instance admin, who receives all permissions. For more information, see [projects members documentation](project/members/index.md). The following table depicts the various user permission levels in a project. @@ -50,7 +50,6 @@ The following table depicts the various user permission levels in a project. |---------------------------------------------------|---------|------------|-------------|----------|--------| | Download project | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | Leave comments | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | -| View Insights charts **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View approved/blacklisted licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View License Compliance reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View Security reports **(ULTIMATE)** | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | @@ -71,6 +70,7 @@ The following table depicts the various user permission levels in a project. | View confidential issues | (*2*) | ✓ | ✓ | ✓ | ✓ | | View [Releases](project/releases/index.md) | ✓ (*6*) | ✓ | ✓ | ✓ | ✓ | | View requirements **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| Manage user-starred metrics dashboards (*7*) | ✓ | ✓ | ✓ | ✓ | ✓ | | Assign issues | | ✓ | ✓ | ✓ | ✓ | | Label issues | | ✓ | ✓ | ✓ | ✓ | | Set issue weight | | ✓ | ✓ | ✓ | ✓ | @@ -83,15 +83,13 @@ The following table depicts the various user permission levels in a project. | See a container registry | | ✓ | ✓ | ✓ | ✓ | | See environments | | ✓ | ✓ | ✓ | ✓ | | See a list of merge requests | | ✓ | ✓ | ✓ | ✓ | -| View project statistics | | ✓ | ✓ | ✓ | ✓ | +| View project statistics | | | ✓ | ✓ | ✓ | | View Error Tracking list | | ✓ | ✓ | ✓ | ✓ | | Create new merge request | | ✓ | ✓ | ✓ | ✓ | | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | | Create/edit requirements **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | -| Pull [packages](packages/index.md) | | ✓ | ✓ | ✓ | ✓ | -| Publish [packages](packages/index.md) | | | ✓ | ✓ | ✓ | -| Pull from [Conan repository](packages/conan_repository/index.md), [Maven repository](packages/maven_repository/index.md), or [NPM registry](packages/npm_registry/index.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | -| Publish to [Conan repository](packages/conan_repository/index.md), [Maven repository](packages/maven_repository/index.md), or [NPM registry](packages/npm_registry/index.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ | +| Pull [packages](packages/index.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | +| Publish [packages](packages/index.md) **(PREMIUM)**| | | ✓ | ✓ | ✓ | | Upload [Design Management](project/issues/design_management.md) files | | | ✓ | ✓ | ✓ | | Create/edit/delete [Releases](project/releases/index.md)| | | ✓ | ✓ | ✓ | | Create new branches | | | ✓ | ✓ | ✓ | @@ -142,20 +140,28 @@ The following table depicts the various user permission levels in a project. | Manage GitLab Pages domains and certificates | | | | ✓ | ✓ | | Remove GitLab Pages | | | | ✓ | ✓ | | Manage clusters | | | | ✓ | ✓ | -| View Pods logs **(ULTIMATE)** | | | | ✓ | ✓ | +| View Pods logs | | | | ✓ | ✓ | | Manage license policy **(ULTIMATE)** | | | | ✓ | ✓ | | Edit comments (posted by any user) | | | | ✓ | ✓ | | Manage Error Tracking | | | | ✓ | ✓ | | Delete wiki pages | | | | ✓ | ✓ | | View project Audit Events | | | | ✓ | ✓ | | Manage [push rules](../push_rules/push_rules.md) | | | | ✓ | ✓ | +| Manage [project access tokens](./project/settings/project_access_tokens.md) **(CORE ONLY)** | | | | ✓ | ✓ | | Switch visibility level | | | | | ✓ | | Transfer project to another namespace | | | | | ✓ | +| Remove fork relationship | | | | | ✓ | | Remove project | | | | | ✓ | | Delete issues | | | | | ✓ | | Disable notification emails | | | | | ✓ | | Force push to protected branches (*4*) | | | | | | | Remove protected branches (*4*) | | | | | | +| View CI\CD analytics | | ✓ | ✓ | ✓ | ✓ | +| View Code Review analytics **(STARTER)** | | ✓ | ✓ | ✓ | ✓ | +| View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Issues analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Repository analytics | | ✓ | ✓ | ✓ | ✓ | +| View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | \* Owner permission is only available at the group or personal namespace level (and for instance admins) and is inherited by its projects. @@ -165,6 +171,7 @@ The following table depicts the various user permission levels in a project. 1. Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [Protected Branches](./project/protected_branches.md). 1. If the [branch is protected](./project/protected_branches.md#using-the-allowed-to-merge-and-allowed-to-push-settings), this depends on the access Developers and Maintainers are given. 1. Guest users can access GitLab [**Releases**](project/releases/index.md) for downloading assets but are not allowed to download the source code nor see repository information like tags and commits. +1. Actions are limited only to records owned (referenced) by user. ## Project features permissions @@ -228,6 +235,8 @@ group. | Create/edit group epic **(ULTIMATE)** | | ✓ | ✓ | ✓ | ✓ | | Manage group labels | | ✓ | ✓ | ✓ | ✓ | | See a container registry | | ✓ | ✓ | ✓ | ✓ | +| Pull [packages](packages/index.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | +| Publish [packages](packages/index.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ | | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | | Create project in group | | | ✓ (3) | ✓ (3) | ✓ (3) | | Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | @@ -244,6 +253,11 @@ group. | Delete group epic **(ULTIMATE)** | | | | | ✓ | | View group Audit Events | | | | | ✓ | | Disable notification emails | | | | | ✓ | +| View Contribution analytics | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Insights **(ULTIMATE)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Issues analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Productivity analytics **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | +| View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | 1. Groups can be set to [allow either Owners or Owners and Maintainers to create subgroups](group/subgroups/index.md#creating-a-subgroup) @@ -391,7 +405,9 @@ instance and project. In addition, all admins can use the admin interface under | See events in the system | | | | ✓ | | Admin interface | | | | ✓ | -1. Only if the job was triggered by the user +1. Only if the job was: + - Triggered by the user + - [Since GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/35069), not run for a protected branch ### Job permissions diff --git a/doc/user/profile/img/change_password_v13_0.png b/doc/user/profile/img/change_password_v13_0.png Binary files differnew file mode 100644 index 00000000000..f63b32557ac --- /dev/null +++ b/doc/user/profile/img/change_password_v13_0.png diff --git a/doc/user/profile/img/unknown_sign_in_email_v13_0.png b/doc/user/profile/img/unknown_sign_in_email_v13_0.png Binary files differnew file mode 100644 index 00000000000..51a7c29cdfa --- /dev/null +++ b/doc/user/profile/img/unknown_sign_in_email_v13_0.png diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index 66ee19437ae..383c7fe73aa 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -17,6 +17,11 @@ There are several ways to create users on GitLab. See the [creating users docume There are several ways to sign into your GitLab account. See the [authentication topic](../../topics/authentication/index.md) for more details. +### Unknown sign-in + +GitLab will notify you if a sign-in occurs that is from an unknown IP address. +See [Unknown Sign-In Notification](unknown_sign_in_notification.md) for more details. + ## User profile To access your profile: @@ -44,6 +49,7 @@ To access your profile settings: From there, you can: - Update your personal information +- Change your [password](#changing-your-password) - Set a [custom status](#current-status) for your profile - Manage your [commit email](#commit-email) for your profile - Manage [2FA](account/two_factor_authentication.md) @@ -60,6 +66,18 @@ From there, you can: - [View your active sessions](active_sessions.md) and revoke any of them if necessary - Access your audit log, a security log of important events involving your account +## Changing your password + +1. Navigate to your [profile's](#profile-settings) **Settings > Password**. +1. Enter your current password in the 'Current password' field. +1. Enter your desired new password twice, once in the 'New password' field and + once in the 'Password confirmation' field. +1. Click the 'Save password' button. + +If you don't know your current password, select the 'I forgot my password' link. + + + ## Changing your username Your `username` is a unique [`namespace`](../group/index.md#namespaces) diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index 1d92f15552d..ae00f3ace57 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -83,6 +83,9 @@ Or: 1. Click the notification dropdown, marked with a bell icon. 1. Select the desired [notification level](#notification-levels). +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a demonstration of how to be notified when a new release is available, see [Notification for releases](https://www.youtube.com/watch?v=qyeNkGgqmH4). + #### Group notifications You can select a notification level and email address for each group. @@ -208,17 +211,17 @@ The following table lists all GitLab-specific email headers: | Header | Description | |------------------------------------|-------------------------------------------------------------------------| -| X-GitLab-Group-Id **(PREMIUM)** | The group's ID. Only present on notification emails for epics. | -| X-GitLab-Group-Path **(PREMIUM)** | The group's path. Only present on notification emails for epics. | -| X-GitLab-Project | The name of the project the notification belongs to. | -| X-GitLab-Project-Id | The project's ID. | -| X-GitLab-Project-Path | The project's path. | -| X-GitLab-(Resource)-ID | The ID of the resource the notification is for. The resource, for example, can be `Issue`, `MergeRequest`, `Commit`, or another such resource. | -| X-GitLab-Discussion-ID | The ID of the thread the comment belongs to, in notification emails for comments. | -| X-GitLab-Pipeline-Id | The ID of the pipeline the notification is for, in notification emails for pipelines. | -| X-GitLab-Reply-Key | A unique token to support reply by email. | -| X-GitLab-NotificationReason | The reason for the notification. This can be `mentioned`, `assigned`, or `own_activity`. | -| List-Id | The path of the project in an RFC 2919 mailing list identifier. This is useful for email organization with filters, for example. | +| `X-GitLab-Group-Id` **(PREMIUM)** | The group's ID. Only present on notification emails for epics. | +| `X-GitLab-Group-Path` **(PREMIUM)** | The group's path. Only present on notification emails for epics. | +| `X-GitLab-Project` | The name of the project the notification belongs to. | +| `X-GitLab-Project-Id` | The project's ID. | +| `X-GitLab-Project-Path` | The project's path. | +| `X-GitLab-(Resource)-ID` | The ID of the resource the notification is for. The resource, for example, can be `Issue`, `MergeRequest`, `Commit`, or another such resource. | +| `X-GitLab-Discussion-ID` | The ID of the thread the comment belongs to, in notification emails for comments. | +| `X-GitLab-Pipeline-Id` | The ID of the pipeline the notification is for, in notification emails for pipelines. | +| `X-GitLab-Reply-Key` | A unique token to support reply by email. | +| `X-GitLab-NotificationReason` | The reason for the notification. This can be `mentioned`, `assigned`, or `own_activity`. | +| `List-Id` | The path of the project in an RFC 2919 mailing list identifier. This is useful for email organization with filters, for example. | ### X-GitLab-NotificationReason diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 1223f7b801a..87c1fe4007a 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -4,15 +4,22 @@ type: concepts, howto # Personal access tokens -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3749) in GitLab 8.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3749) in GitLab 8.8. +> - [Notifications about expiring tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) added in GitLab 12.6. +> - [Token lifetime limits](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) added in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. -If you're unable to use [OAuth2](../../api/oauth2.md), you can use a personal access token to authenticate with the [GitLab API](../../api/README.md#personal-access-tokens). +If you're unable to use [OAuth2](../../api/oauth2.md), you can use a personal access token to authenticate with the [GitLab API](../../api/README.md#personalproject-access-tokens). You can also use personal access tokens with Git to authenticate over HTTP or SSH. Personal access tokens are required when [Two-Factor Authentication (2FA)](../account/two_factor_authentication.md) is enabled. In both cases, you can authenticate with a token in place of your password. Personal access tokens expire on the date you define, at midnight UTC. -For examples of how you can use a personal access token to authenticate with the API, see the following section from our [API Docs](../../api/README.md#personal-access-tokens). +- GitLab runs a check at 01:00 AM UTC every day to identify personal access tokens that will expire in under seven days. The owners of these tokens are notified by email. +- In GitLab Ultimate, administrators may [limit the lifetime of personal access tokens](../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only). + +For examples of how you can use a personal access token to authenticate with the API, see the following section from our [API Docs](../../api/README.md#personalproject-access-tokens). + +GitLab also offers [impersonation tokens](../../api/README.md#impersonation-tokens) which are created by administrators via the API. They're a great fit for automated authentication as a specific user. ## Creating a personal access token diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index cd195e6e7a1..55781b48a27 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -55,6 +55,11 @@ The default syntax theme is White, and you can choose among 5 different themes:  +[Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2389) in 13.0, the theme +you choose also applies to the [Web IDE](../project/web_ide/index.md)'s code editor and [Snippets](../snippets.md). +The themes are available only in the Web IDE file editor, except for the [dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/209808), +which applies to the entire Web IDE screen. + ## Behavior The following settings allow you to customize the behavior of GitLab's layout diff --git a/doc/user/profile/unknown_sign_in_notification.md b/doc/user/profile/unknown_sign_in_notification.md new file mode 100644 index 00000000000..9400ead1922 --- /dev/null +++ b/doc/user/profile/unknown_sign_in_notification.md @@ -0,0 +1,16 @@ +# Email notification for unknown sign-ins + +When a user successfully signs in from a previously unknown IP address, +GitLab notifies the user by email. In this way, GitLab proactively alerts users of potentially +malicious or unauthorized sign-ins. + +There are two methods used to identify a known sign-in: + +- Last sign-in IP: The current sign-in IP address is checked against the last sign-in + IP address. +- Current active sessions: If the user has an existing active session from the + same IP address. See [Active Sessions](active_sessions.md). + +## Example email + + diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index 7fa8ec6c5f3..712f8ea0adc 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Adding EKS clusters GitLab supports adding new and existing EKS clusters. diff --git a/doc/user/project/clusters/add_gke_clusters.md b/doc/user/project/clusters/add_gke_clusters.md index 1195421f8fb..4094828323a 100644 --- a/doc/user/project/clusters/add_gke_clusters.md +++ b/doc/user/project/clusters/add_gke_clusters.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Adding GKE clusters GitLab supports adding new and existing GKE clusters. diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index dce273ce602..fddc9873f17 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Adding and removing Kubernetes clusters GitLab offers integrated cluster creation for the following Kubernetes providers: diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 74a58b93442..1298a24fcac 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Kubernetes clusters > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/35954) in GitLab 10.1 for projects. @@ -30,10 +36,28 @@ Using the GitLab project Kubernetes integration, you can: - View [Logs](#logs). - Run serverless workloads on [Kubernetes with Knative](serverless/index.md). +### Supported cluster versions + +GitLab is committed to support at least two production-ready Kubernetes minor versions at any given time. We regularly review the versions we support, and provide a four-month deprecation period before we remove support of a specific version. The range of supported versions is based on the evaluation of: + +- Our own needs. +- The versions supported by major managed Kubernetes providers. +- The versions [supported by the Kubernetes community](https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-versions). + +Currently, GitLab supports the following Kubernetes versions: + +- 1.15 +- 1.14 +- 1.13 (deprecated, support ends on November 22, 2020) +- 1.12 (deprecated, support ends on September 22, 2020) + +NOTE: **Note:** +Some GitLab features may support versions outside the range provided here. + ### Deploy Boards **(PREMIUM)** GitLab's Deploy Boards offer a consolidated view of the current health and -status of each CI [environment](../../../ci/environments.md) running on Kubernetes, +status of each CI [environment](../../../ci/environments/index.md) running on Kubernetes, displaying the status of the pods in the deployment. Developers and other teammates can view the progress and status of a rollout, pod by pod, in the workflow they already use without any need to access Kubernetes. @@ -78,8 +102,8 @@ Kubernetes clusters can be used without Auto DevOps. > Introduced in GitLab 8.15. -When enabled, the Kubernetes integration adds [web terminal](../../../ci/environments.md#web-terminals) -support to your [environments](../../../ci/environments.md). This is based on the `exec` functionality found in +When enabled, the Kubernetes integration adds [web terminal](../../../ci/environments/index.md#web-terminals) +support to your [environments](../../../ci/environments/index.md). This is based on the `exec` functionality found in Docker and Kubernetes, so you get a new shell session within your existing containers. To use this integration, you should deploy to Kubernetes using the deployment variables above, ensuring any deployments, replica sets, and @@ -181,8 +205,8 @@ you can either: ### Setting the environment scope **(PREMIUM)** When adding more than one Kubernetes cluster to your project, you need to differentiate -them with an environment scope. The environment scope associates clusters with [environments](../../../ci/environments.md) similar to how the -[environment-specific variables](../../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables) work. +them with an environment scope. The environment scope associates clusters with [environments](../../../ci/environments/index.md) similar to how the +[environment-specific variables](../../../ci/variables/README.md#limit-the-environment-scopes-of-environment-variables) work. The default environment scope is `*`, which means all jobs, regardless of their environment, will use that cluster. Each scope can only be used by a single @@ -262,7 +286,7 @@ A Kubernetes cluster can be the destination for a deployment job. If the cluster from your jobs using tools such as `kubectl` or `helm`. - You don't use GitLab's cluster integration you can still deploy to your cluster. However, you will need configure Kubernetes tools yourself - using [environment variables](../../../ci/variables/README.md#creating-a-custom-environment-variable) + using [environment variables](../../../ci/variables/README.md#custom-environment-variables) before you can interact with the cluster from your jobs. ### Deployment variables @@ -297,7 +321,7 @@ of the form `<project_name>-<project_id>-<environment>` (see [Deployment variables](#deployment-variables)). For **non**-GitLab-managed clusters, the namespace can be customized using -[`environment:kubernetes:namespace`](../../../ci/environments.md#configuring-kubernetes-deployments) +[`environment:kubernetes:namespace`](../../../ci/environments/index.md#configuring-kubernetes-deployments) in `.gitlab-ci.yml`. NOTE: **Note:** When using a [GitLab-managed cluster](#gitlab-managed-clusters), the @@ -314,7 +338,7 @@ the deployment job: However, sometimes GitLab can not create them. In such instances, your job will fail with the message: -```text +```plaintext This job failed because the necessary resources were not successfully created. ``` @@ -325,7 +349,7 @@ Reasons for failure include: - The token you gave GitLab does not have [`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) privileges required by GitLab. - Missing `KUBECONFIG` or `KUBE_TOKEN` variables. To be passed to your job, they must have a matching - [`environment:name`](../../../ci/environments.md#defining-environments). If your job has no + [`environment:name`](../../../ci/environments/index.md#defining-environments). If your job has no `environment:name` set, it will not be passed the Kubernetes credentials. NOTE: **NOTE:** diff --git a/doc/user/project/clusters/kubernetes_pod_logs.md b/doc/user/project/clusters/kubernetes_pod_logs.md index 5543187b6de..8577231b69b 100644 --- a/doc/user/project/clusters/kubernetes_pod_logs.md +++ b/doc/user/project/clusters/kubernetes_pod_logs.md @@ -1,24 +1,36 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Kubernetes Logs > - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4752) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26383) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. GitLab makes it easy to view the logs of running pods in [connected Kubernetes clusters](index.md). -By displaying the logs directly in GitLab, developers can avoid having to manage console tools or jump to a different interface. +By displaying the logs directly in GitLab in the **Log Explorer**, developers can avoid +managing console tools or jumping to a different interface. NOTE: **Kubernetes + GitLab** -Everything you need to build, test, deploy, and run your app at scale. +Everything you need to build, test, deploy, and run your application at scale. [Learn more](https://about.gitlab.com/solutions/kubernetes/). ## Overview -[Kubernetes](https://kubernetes.io) logs can be viewed directly within GitLab. +[Kubernetes](https://kubernetes.io) logs can be viewed directly within GitLab with +the **Log Explorer**.  +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +To learn more, see [APM - Log Explorer](https://www.youtube.com/watch?v=hWclZHA7Dgw). + ## Requirements -[Deploying to a Kubernetes environment](../deploy_boards.md#enabling-deploy-boards) is required in order to be able to use Logs. +[Deploying to a Kubernetes environment](../deploy_boards.md#enabling-deploy-boards) +is required to use Logs. ## Usage @@ -30,7 +42,8 @@ You can access them in two ways. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22011) in GitLab 12.5. -Go to **{cloud-gear}** **Operations > Logs** on the sidebar menu. +Go to **{cloud-gear}** **Operations > Pod logs** on the sidebar menu to display +the **Log Explorer**.  @@ -38,34 +51,42 @@ Go to **{cloud-gear}** **Operations > Logs** on the sidebar menu. Logs can be displayed by clicking on a specific pod from [Deploy Boards](../deploy_boards.md): -1. Go to **{cloud-gear}** **Operations > Environments** and find the environment which contains the desired pod, like `production`. -1. On the **Environments** page, you should see the status of the environment's pods with [Deploy Boards](../deploy_boards.md). -1. When mousing over the list of pods, a tooltip will appear with the exact pod name and status. +1. Go to **{cloud-gear}** **Operations > Environments** and find the environment + which contains the desired pod, like `production`. +1. On the **Environments** page, you should see the status of the environment's + pods with [Deploy Boards](../deploy_boards.md). +1. When mousing over the list of pods, a tooltip will appear with the exact pod name + and status.  -1. Click on the desired pod to bring up the logs view. +1. Click on the desired pod to display the **Log Explorer**. ### Logs view -The logs view lets you filter the logs by: +The **Log Explorer** lets you filter the logs by: - Pods. - [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/5769), environments. -- [From GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21656), [full text search](#full-text-search). +- [From GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21656), + [full text search](#full-text-search). - [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/197879), dates. -Loading more than 500 log lines is possible from [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/198050) onwards. +Loading more than 500 log lines is possible from +[GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/198050) onward. -Support for pods with multiple containers is coming [in a future release](https://gitlab.com/gitlab-org/gitlab/issues/13404). +Support for pods with multiple containers is coming +[in a future release](https://gitlab.com/gitlab-org/gitlab/issues/13404). -Support for historical data is coming [in a future release](https://gitlab.com/gitlab-org/gitlab/issues/196191). +Support for historical data is coming +[in a future release](https://gitlab.com/gitlab-org/gitlab/issues/196191). ### Filter by date > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197879) in GitLab 12.8. -When you enable [Elastic Stack](../../clusters/applications.md#elastic-stack) on your cluster, you can filter by date. +When you enable [Elastic Stack](../../clusters/applications.md#elastic-stack) +on your cluster, you can filter logs displayed in the **Log Explorer** by date. -Click on **Show last** to see the available options. +Click **Show last** in the **Log Explorer** to see the available options. ### Full text search @@ -74,7 +95,8 @@ Click on **Show last** to see the available options. When you enable [Elastic Stack](../../clusters/applications.md#elastic-stack) on your cluster, you can search the content of your logs through a search bar. -The search is passed on to Elasticsearch using the [simple_query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html) +The search is passed on to Elasticsearch using the +[simple_query_string](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html) Elasticsearch function, which supports the following operators: | Operator | Description | diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index 5575cd1d2d4..dfed43470bc 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -1,14 +1,19 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Runbooks Runbooks are a collection of documented procedures that explain how to carry out a particular process, be it starting, stopping, debugging, or troubleshooting a particular system. -Using [Jupyter Notebooks](https://jupyter.org/) and the [Rubix library](https://github.com/Nurtch/rubix), +Using [Jupyter Notebooks](https://jupyter.org/) and the +[Rubix library](https://github.com/Nurtch/rubix), users can get started writing their own executable runbooks. -## Overview - Historically, runbooks took the form of a decision tree or a detailed step-by-step guide depending on the condition or system. @@ -22,121 +27,128 @@ pre-written code blocks or database queries against a given environment. The JupyterHub app offered via GitLab’s Kubernetes integration now ships with Nurtch’s Rubix library, providing a simple way to create DevOps -runbooks. A sample runbook is provided, showcasing common operations. While Rubix makes it -simple to create common Kubernetes and AWS workflows, you can also create them manually without -Rubix. +runbooks. A sample runbook is provided, showcasing common operations. While +Rubix makes it simple to create common Kubernetes and AWS workflows, you can +also create them manually without Rubix. -**<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch this [video](https://www.youtube.com/watch?v=Q_OqHIIUPjE) -for an overview of how this is accomplished in GitLab!** +for an overview of how this is accomplished in GitLab! ## Requirements To create an executable runbook, you will need: -1. **Kubernetes** - A Kubernetes cluster is required to deploy the rest of the applications. - The simplest way to get started is to add a cluster using one of [GitLab's integrations](../add_remove_clusters.md#create-new-cluster). -1. **Helm Tiller** - Helm is a package manager for Kubernetes and is required to install - all the other applications. It is installed in its own pod inside the cluster which - can run the Helm CLI in a safe environment. -1. **Ingress** - Ingress can provide load balancing, SSL termination, and name-based - virtual hosting. It acts as a web proxy for your applications. -1. **JupyterHub** - [JupyterHub](https://jupyterhub.readthedocs.io/) is a multi-user service for managing notebooks across - a team. Jupyter Notebooks provide a web-based interactive programming environment - used for data analysis, visualization, and machine learning. +- **Kubernetes** - A Kubernetes cluster is required to deploy the rest of the + applications. The simplest way to get started is to add a cluster using one + of [GitLab's integrations](../add_remove_clusters.md#create-new-cluster). +- **Helm Tiller** - Helm is a package manager for Kubernetes and is required to + install all the other applications. It's installed in its own pod inside the + cluster which can run the Helm CLI in a safe environment. +- **Ingress** - Ingress can provide load balancing, SSL termination, and name-based + virtual hosting. It acts as a web proxy for your applications. +- **JupyterHub** - [JupyterHub](https://jupyterhub.readthedocs.io/) is a multi-user + service for managing notebooks across a team. Jupyter Notebooks provide a + web-based interactive programming environment used for data analysis, + visualization, and machine learning. ## Nurtch -Nurtch is the company behind the [Rubix library](https://github.com/Nurtch/rubix). Rubix is -an open-source Python library that makes it easy to perform common DevOps tasks inside Jupyter Notebooks. -Tasks such as plotting Cloudwatch metrics and rolling your ECS/Kubernetes app are simplified -down to a couple of lines of code. See the [Nurtch Documentation](http://docs.nurtch.com/en/latest/) -for more information. +Nurtch is the company behind the [Rubix library](https://github.com/Nurtch/rubix). +Rubix is an open-source Python library that makes it easy to perform common +DevOps tasks inside Jupyter Notebooks. Tasks such as plotting Cloudwatch metrics +and rolling your ECS/Kubernetes app are simplified down to a couple of lines of +code. See the [Nurtch Documentation](http://docs.nurtch.com/en/latest/) for more +information. ## Configure an executable runbook with GitLab Follow this step-by-step guide to configure an executable runbook in GitLab using -the components outlined above and the preloaded demo runbook. - -### 1. Add a Kubernetes cluster - -Follow the steps outlined in [Create new cluster](../add_remove_clusters.md#create-new-cluster) -to add a Kubernetes cluster to your project. - -### 2. Install Helm Tiller, Ingress, and JupyterHub - -Once the cluster has been provisioned in GKE, click the **Install** button next to the **Helm Tiller** app. - - +the components outlined above and the pre-loaded demo runbook. -Once Tiller has been installed successfully, click the **Install** button next to the **Ingress** app. +1. Add a Kubernetes cluster to your project by following the steps outlined in + [Create new cluster](../add_remove_clusters.md#create-new-cluster). +1. After the cluster has been provisioned in GKE, click the **Install** button + next to the **Helm Tiller** application to install Helm Tiller. - +  -Once Ingress has been installed successfully, click the **Install** button next to the **JupyterHub** app. +1. After Helm Tiller has been installed successfully, click the **Install** button next + to the **Ingress** application. - +  -### 3. Login to JupyterHub and start the server +1. After Ingress has been installed successfully, click the **Install** button next + to the **JupyterHub** application. You will need the **Jupyter Hostname** provided + here in the next step. -Once JupyterHub has been installed successfully, navigate to the displayed **Jupyter Hostname** URL and click -**Sign in with GitLab**. Authentication is automatically enabled for any user of the GitLab instance via OAuth2. This -will redirect to GitLab in order to authorize JupyterHub to use your GitLab account. Click **Authorize**. +  - +1. After JupyterHub has been installed successfully, open the **Jupyter Hostname** + in your browser. Click the **Sign in with GitLab** button to log in to + JupyterHub and start the server. Authentication is enabled for any user of the + GitLab instance with OAuth2. This button redirects you to a page at GitLab + requesting authorization for JupyterHub to use your GitLab account. -Once the application has been authorized you will taken back to the JupyterHub application. Click **Start My Server**. -The server will take a couple of seconds to start. +  -### 4. Configure access +1. Click **Authorize**, and you will be redirected to the JupyterHub application. +1. Click **Start My Server**, and the server will start in a few seconds. +1. To configure the runbook's access to your GitLab project, you must enter your + [GitLab Access Token](../../../profile/personal_access_tokens.md) + and your Project ID in the **Setup** section of the demo runbook: -In order for the runbook to access your GitLab project, you will need to enter a -[GitLab Access Token](../../../profile/personal_access_tokens.md) -as well as your Project ID in the **Setup** section of the demo runbook. + 1. Double-click the **DevOps-Runbook-Demo** folder located on the left panel. -Double-click the **DevOps-Runbook-Demo** folder located on the left panel. +  - + 1. Double-click the `Nurtch-DevOps-Demo.ipynb` runbook. -Double-click the "Nurtch-DevOps-Demo.ipynb" runbook. +  - + Jupyter displays the runbook's contents in the right-hand side of the screen. + The **Setup** section displays your `PRIVATE_TOKEN` and your `PROJECT_ID`. + Enter these values, maintaining the single quotes as follows: -The contents on the runbook will be displayed on the right side of the screen. Under the "Setup" section, you will find -entries for both your `PRIVATE_TOKEN` and your `PROJECT_ID`. Enter both these values, conserving the single quotes as follows: + ```sql + PRIVATE_TOKEN = 'n671WNGecHugsdEDPsyo' + PROJECT_ID = '1234567' + ``` -```sql -PRIVATE_TOKEN = 'n671WNGecHugsdEDPsyo' -PROJECT_ID = '1234567' -``` + 1. Update the `VARIABLE_NAME` on the last line of this section to match the name of + the variable you're using for your access token. In this example, our variable + name is `PRIVATE_TOKEN`. -Update the `VARIABLE_NAME` on the last line of this section to match the name of the variable you are using for your -access token. In this example our variable name is `PRIVATE_TOKEN`. + ```sql + VARIABLE_VALUE = project.variables.get('PRIVATE_TOKEN').value + ``` -```sql -VARIABLE_VALUE = project.variables.get('PRIVATE_TOKEN').value -``` +1. To configure the operation of a runbook, create and configure variables: -### 5. Configure an operation + NOTE: **Note:** + For this example, we are using the **Run SQL queries in Notebook** section in the + sample runbook to query a PostgreSQL database. The first four lines of the following + code block define the variables that are required for this query to function: -For this example we'll use the "**Run SQL queries in Notebook**" section in the sample runbook to query -a PostgreSQL database. The first 4 lines of the section define the variables that are required for this query to function. + ```sql + %env DB_USER={project.variables.get('DB_USER').value} + %env DB_PASSWORD={project.variables.get('DB_PASSWORD').value} + %env DB_ENDPOINT={project.variables.get('DB_ENDPOINT').value} + %env DB_NAME={project.variables.get('DB_NAME').value} + ``` -```sql -%env DB_USER={project.variables.get('DB_USER').value} -%env DB_PASSWORD={project.variables.get('DB_PASSWORD').value} -%env DB_ENDPOINT={project.variables.get('DB_ENDPOINT').value} -%env DB_NAME={project.variables.get('DB_NAME').value} -``` + 1. Navigate to **{settings}** **Settings >> CI/CD >> Variables** to create + the variables in your project. -Create the matching variables in your project's **Settings >> CI/CD >> Variables** +  - + 1. Click **Save variables**. -Back in Jupyter, click the "Run SQL queries in Notebook" heading and the click *Run*. The results will be -displayed in-line as follows: + 1. In Jupyter, click the **Run SQL queries in Notebook** heading, and then click + **Run**. The results are displayed inline as follows: - +  -You can try other operations such as running shell scripts or interacting with a Kubernetes cluster. Visit the +You can try other operations, such as running shell scripts or interacting with a +Kubernetes cluster. Visit the [Nurtch Documentation](http://docs.nurtch.com/) for more information. diff --git a/doc/user/project/clusters/serverless/aws.md b/doc/user/project/clusters/serverless/aws.md index 3df57e3a7a5..124a0d4bf9f 100644 --- a/doc/user/project/clusters/serverless/aws.md +++ b/doc/user/project/clusters/serverless/aws.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Deploying AWS Lambda function using GitLab CI/CD GitLab allows users to easily deploy AWS Lambda functions and create rich serverless applications. @@ -150,7 +156,7 @@ endpoints: Running the following `curl` command should trigger your function. NOTE: **Note:** - Your url should be the one retrieved from the GitLab deploy stage log. +Your URL should be the one retrieved from the GitLab deploy stage log. ```shell curl https://u768nzby1j.execute-api.us-east-1.amazonaws.com/production/hello diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index 418e16aa0c1..2156d96f92a 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -1,3 +1,9 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Serverless > Introduced in GitLab 11.5. diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 49083be77dc..45d9e8f04e0 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -9,6 +9,33 @@ in [GitLab Starter](https://about.gitlab.com/pricing/) 11.3. > - [Support for group namespaces](https://gitlab.com/gitlab-org/gitlab-foss/issues/53182) added in GitLab Starter 12.1. > - Code Owners for Merge Request approvals was [introduced](https://gitlab.com/gitlab-org/gitlab/issues/4418) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.9. +## Introduction + +When contributing to a project, it can often be difficult +to find out who should review or approve merge requests. +Additionally, if you have a question over a specific file or +code block, it may be difficult to know who to find the answer from. + +GitLab Code Owners is a feature to define who owns specific +files or paths in a repository, allowing other users to understand +who is responsible for each file or path. + +## Why is this useful? + +Code Owners allows for a version controlled single source of +truth file outlining the exact GitLab users or groups that +own certain files or paths in a repository. Code Owners can be +utilized in the merge request approval process which can streamline +the process of finding the right reviewers and approvers for a given +merge request. + +In larger organizations or popular open source projects, Code Owners +can also be useful to understand who to contact if you have +a question that may not be related to code review or a merge request +approval. + +## How to set up Code Owners + You can use a `CODEOWNERS` file to specify users or [shared groups](members/share_project_with_groups.md) that are responsible for certain files in a repository. @@ -41,7 +68,7 @@ The user that would show for `README.md` would be `@user2`. ## Approvals by Code Owners Once you've set Code Owners to a project, you can configure it to -receive approvals: +be used for merge request approvals: - As [merge request eligible approvers](merge_requests/merge_request_approvals.md#code-owners-as-eligible-approvers). - As required approvers for [protected branches](protected_branches.md#protected-branches-approval-by-code-owners-premium). **(PREMIUM)** @@ -50,6 +77,9 @@ Once set, Code Owners are displayed in merge requests widgets:  +NOTE: **Note**: + While the`CODEOWNERS` file can be used in addition to Merge Request [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules) it can also be used as the sole driver of a Merge Request approval (without using [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules)) by simply creating the file in one of the three locations specified above, configuring the Code Owners to be required approvers for [protected branches](protected_branches.md#protected-branches-approval-by-code-owners-premium) and then using [the syntax of Code Owners files](code_owners.md#the-syntax-of-code-owners-files) to specify the actual owners and granular permissions. + ## The syntax of Code Owners files Files can be specified using the same kind of patterns you would use @@ -69,23 +99,28 @@ escaped using `\#` to address files for which the name starts with a Example `CODEOWNERS` file: ```plaintext -# This is an example code owners file, lines starting with a `#` will -# be ignored. +# This is an example of a code owners file +# lines starting with a `#` will be ignored. # app/ @commented-rule # We can specify a default match using wildcards: * @default-codeowner +# We can also specify "multiple tab or space" separated codeowners: +* @multiple @code @owners + # Rules defined later in the file take precedence over the rules # defined before. # This will match all files for which the file name ends in `.rb` *.rb @ruby-owner -# Files with a `#` can still be accesssed by escaping the pound sign +# Files with a `#` can still be accessed by escaping the pound sign \#file_with_pound.rb @owner-file-with-pound # Multiple codeowners can be specified, separated by spaces or tabs +# In the following case the CODEOWNERS file from the root of the repo +# has 3 code owners (@multiple @code @owners) CODEOWNERS @multiple @code @owners # Both usernames or email addresses can be used to match diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index c479f610ff1..8f7bb844e37 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -3,7 +3,7 @@ > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1589) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. GitLab's Deploy Boards offer a consolidated view of the current health and -status of each CI [environment](../../ci/environments.md) running on [Kubernetes](https://kubernetes.io), displaying the status +status of each CI [environment](../../ci/environments/index.md) running on [Kubernetes](https://kubernetes.io), displaying the status of the pods in the deployment. Developers and other teammates can view the progress and status of a rollout, pod by pod, in the workflow they already use without any need to access Kubernetes. @@ -57,9 +57,9 @@ specific environment, there are a lot of use cases. To name a few: ## Enabling Deploy Boards -To display the Deploy Boards for a specific [environment](../../ci/environments.md) you should: +To display the Deploy Boards for a specific [environment](../../ci/environments/index.md) you should: -1. Have [defined an environment](../../ci/environments.md#defining-environments) with a deploy stage. +1. Have [defined an environment](../../ci/environments/index.md#defining-environments) with a deploy stage. 1. Have a Kubernetes cluster up and running. @@ -113,7 +113,7 @@ metadata: name: "APPLICATION_NAME" annotations: app.gitlab.com/app: ${CI_PROJECT_PATH_SLUG} - app.gitlab.com/env: ${CI_ENVIRONMENT_SLUG} + app.gitlab.com/env: ${CI_ENVIRONMENT_SLUG} spec: replicas: 1 selector: @@ -130,6 +130,11 @@ spec: The annotations will be applied to the deployments, replica sets, and pods. By changing the number of replicas, like `kubectl scale --replicas=3 deploy APPLICATION_NAME -n ${KUBE_NAMESPACE}`, you can follow the instances' pods from the board. +NOTE: **Note:** +The YAML file is static. If you apply it using `kubectl apply`, you must +manually provide the project and environment slugs, or create a script to +replace the variables in the YAML before applying. + ## Canary Deployments A popular CI strategy, where a small portion of the fleet is updated to the new @@ -141,5 +146,5 @@ version of your application. - [GitLab Autodeploy](../../topics/autodevops/stages.md#auto-deploy) - [GitLab CI/CD environment variables](../../ci/variables/README.md) -- [Environments and deployments](../../ci/environments.md) +- [Environments and deployments](../../ci/environments/index.md) - [Kubernetes deploy example](https://gitlab.com/gitlab-examples/kubernetes-deploy) diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index ebb12a6ed5d..2d42debed68 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -3,8 +3,10 @@ > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17894) in GitLab 10.7. > - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/199370) from **Settings > Repository** in GitLab 12.9. > - [Added `write_registry` scope](https://gitlab.com/gitlab-org/gitlab/-/issues/22743) in GitLab 12.10. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29280) from **Settings > CI / CD** in GitLab 12.10.1. +> - [Added package registry scopes](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) from **Settings > CI / CD** in GitLab 13.0. -Deploy tokens allow you to download (`git clone`) or push and pull the container registry images of a project without having a user and a password. +Deploy tokens allow you to download (`git clone`) or push and pull packages and container registry images of a project without having a user and a password. Deploy tokens can be managed by [maintainers only](../../permissions.md). @@ -16,7 +18,7 @@ You can create as many deploy tokens as you like from the settings of your proje 1. Log in to your GitLab account. 1. Go to the project (or group) you want to create Deploy Tokens for. -1. Go to **{settings}** **Settings** > **CI / CD**. +1. Go to **{settings}** **Settings** > **Repository**. 1. Click on "Expand" on **Deploy Tokens** section. 1. Choose a name, expiry date (optional), and username (optional) for the token. 1. Choose the [desired scopes](#limiting-scopes-of-a-deploy-token). @@ -65,7 +67,7 @@ To download a repository using a Deploy Token, you just need to: 1. `git clone` the project using the Deploy Token: ```shell - git clone http://<username>:<deploy_token>@gitlab.example.com/tanuki/awesome_project.git + git clone https://<username>:<deploy_token>@gitlab.example.com/tanuki/awesome_project.git ``` Replace `<username>` and `<deploy_token>` with the proper values. @@ -100,6 +102,22 @@ To push the container registry images, you'll need to: Just replace `<username>` and `<deploy_token>` with the proper values. Then you can simply push images to your Container Registry. +### Read or pull packages + +To pull packages in the GitLab package registry, you'll need to: + +1. Create a Deploy Token with `read_package_registry` as a scope. +1. Take note of your `username` and `token`. +1. For the [package type of your choice](./../../packages/index.md), follow the authentication instructions for deploy tokens. + +### Push or upload packages + +To upload packages in the GitLab package registry, you'll need to: + +1. Create a Deploy Token with `write_package_registry` as a scope. +1. Take note of your `username` and `token`. +1. For the [package type of your choice](./../../packages/index.md), follow the authentication instructions for deploy tokens. + ### Group Deploy Token > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21765) in GitLab 12.9. @@ -107,6 +125,9 @@ push images to your Container Registry. A deploy token created at the group level can be used across all projects that belong either to the specific group or to one of its subgroups. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [Group Deploy Tokens](https://youtu.be/8kxTJvaD9ks). + To use a group deploy token: 1. [Create](#creating-a-deploy-token) a deploy token for a group. @@ -132,3 +153,6 @@ those variables: ```shell docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY ``` + +NOTE: **Note:** +The special handling for the `gitlab-deploy-token` deploy token is not currently implemented for group deploy tokens. For the deploy token to be available for CI/CD jobs, it must be created at the project level. See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/214014) for details. diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index a02dc016f03..16ac53a2b52 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -39,6 +39,26 @@ templates of the default branch will be taken into account. Create a new Markdown (`.md`) file inside the `.gitlab/issue_templates/` directory in your repository. Commit and push to your default branch. +To create a Markdown file: + + 1. Click the `+` button next to `master` and click **New file**. + 1. Add the name of your issue template to the **File name** text field next to `master`. + Make sure words are separated with underscores and that your file has the `.md` extension, for + example `feature_request.md`. + 1. Commit and push to your default branch. + +If you don't have a `.gitlab/issue_templates` directory in your repository, you'll need to create it. + +To create the `.gitlab/issue_templates` directory: + + 1. Click the `+` button next to `master` and select **New directory**. + 1. Name this new directory `.gitlab` and commit to your default branch. + 1. Click the `+` button next to `master` again and select **New directory**.This time, n + 1. Name your directory `issue_templates` and commit to your default branch. + +To check if this has worked correctly, [create a new issue](./issues/managing_issues.md#create-a-new-issue) +and see if you can choose a description template. + ## Creating merge request templates Similarly to issue templates, create a new Markdown (`.md`) file inside the diff --git a/doc/user/project/file_lock.md b/doc/user/project/file_lock.md index d5f35051e9a..9069a231db4 100644 --- a/doc/user/project/file_lock.md +++ b/doc/user/project/file_lock.md @@ -2,9 +2,10 @@ > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/440) in [GitLab Premium](https://about.gitlab.com/pricing/) 8.9. -File Locking helps you avoid merge conflicts and better manage your binary files. -Lock any file or directory, make your changes, and then unlock it so another -member of the team can edit it. +Working with multiple people on the same file can be a risk. Conflicts when merging a non-text file are hard to overcome and will require a lot of manual work to resolve. File Locking helps you avoid these merge conflicts and better manage your binary files. + +With File Locking, you can lock any file or directory, make your changes, and +then unlock it so another member of the team can edit it. ## Overview diff --git a/doc/user/project/git_attributes.md b/doc/user/project/git_attributes.md index 21ef94e61f7..260e618ba03 100644 --- a/doc/user/project/git_attributes.md +++ b/doc/user/project/git_attributes.md @@ -1,6 +1,6 @@ # Git Attributes -GitLab supports defining custom [Git attributes][gitattributes] such as what +GitLab supports defining custom [Git attributes](https://git-scm.com/docs/gitattributes) such as what files to treat as binary, and what language to use for syntax highlighting diffs. @@ -18,5 +18,3 @@ ignored. The `.gitattributes` file can be used to define which language to use when syntax highlighting files and diffs. See ["Syntax Highlighting"](highlighting.md) for more information. - -[gitattributes]: https://git-scm.com/docs/gitattributes diff --git a/doc/user/project/img/service_desk_custom_email_address_v13_0.png b/doc/user/project/img/service_desk_custom_email_address_v13_0.png Binary files differnew file mode 100644 index 00000000000..6ce8bf45085 --- /dev/null +++ b/doc/user/project/img/service_desk_custom_email_address_v13_0.png diff --git a/doc/user/project/import/bitbucket.md b/doc/user/project/import/bitbucket.md index 77fc2761e07..56717858b53 100644 --- a/doc/user/project/import/bitbucket.md +++ b/doc/user/project/import/bitbucket.md @@ -24,7 +24,7 @@ Import your projects from Bitbucket Cloud to GitLab with minimal effort. ## Requirements -The [Bitbucket Cloud integration][bb-import] must be first enabled in order to be +The [Bitbucket Cloud integration](../../../integration/bitbucket.md) must be first enabled in order to be able to import your projects from Bitbucket Cloud. Ask your GitLab administrator to enable this if not already. @@ -42,7 +42,7 @@ The importer will create any new namespaces (groups) if they don't exist or in the case the namespace is taken, the repository will be imported under the user's namespace that started the import process. -## Importing your Bitbucket repositories +## Import your Bitbucket repositories 1. Sign in to GitLab and go to your dashboard. 1. Click on **New project**. @@ -61,5 +61,11 @@ namespace that started the import process.  -[bb-import]: ../../../integration/bitbucket.md -[social sign-in]: ../../profile/account/social_sign_in.md +## Troubleshooting + +If you have more than one Bitbucket account, be sure to sign in to the correct account. +If you've accidentally started the import process with the wrong account, follow these steps: + +1. Revoke GitLab access to your Bitbucket account, essentially reversing the process in the following procedure: [Import your Bitbucket repositories](#import-your-bitbucket-repositories). + +1. Sign out of the Bitbucket account. Follow the procedure linked from the previous step. diff --git a/doc/user/project/import/bitbucket_server.md b/doc/user/project/import/bitbucket_server.md index 99179c3190e..55df2d7294d 100644 --- a/doc/user/project/import/bitbucket_server.md +++ b/doc/user/project/import/bitbucket_server.md @@ -73,3 +73,7 @@ namespace that started the import process. imported.  + +## Troubleshooting + +See the [troubleshooting](bitbucket.md#troubleshooting) section for [Bitbucket](bitbucket.md). diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index db8d33b58da..4c213f21920 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -8,7 +8,7 @@ your self-managed GitLab instance. NOTE: **Note:** These instructions work for users on GitLab.com, but if you are an administrator of a self-managed GitLab instance or if you are importing from GitHub Enterprise, -you must enable [GitHub integration][gh-import]. GitHub integration is the only method for +you must enable [GitHub integration](../../../integration/github.md). GitHub integration is the only method for importing from GitHub Enterprise. If you are using GitLab.com, you can alternatively import GitHub repositories using a [personal access token](#using-a-github-token), but this method is not recommended because it cannot associate all user activity @@ -81,7 +81,7 @@ the user account that is performing the import. NOTE: **Note:** If you are using a self-managed GitLab instance or if you are importing from GitHub Enterprise, this process requires that you have configured -[GitHub integration][gh-import]. +[GitHub integration](../../../integration/github.md). 1. From the top navigation bar, click **+** and select **New project**. 1. Select the **Import project** tab and then select **GitHub**. @@ -155,5 +155,3 @@ servers. For 4 servers with 8 cores this means you can import up to 32 objects ( Reducing the time spent in cloning a repository can be done by increasing network throughput, CPU capacity, and disk performance (e.g., by using high performance SSDs) of the disks that store the Git repositories (for your GitLab instance). Increasing the number of Sidekiq workers will *not* reduce the time spent cloning repositories. - -[gh-import]: ../../../integration/github.md "GitHub integration" diff --git a/doc/user/project/import/jira.md b/doc/user/project/import/jira.md index 49224001fe6..db48282a8f3 100644 --- a/doc/user/project/import/jira.md +++ b/doc/user/project/import/jira.md @@ -9,6 +9,15 @@ Jira issues import is an MVC, project-level feature, meaning that issues from mu Jira projects can be imported into a GitLab project. MVC version imports issue title and description as well as some other issue metadata as a section in the issue description. +## Future iterations + +As of GitLab 12.10, the Jira issue importer only brings across the title and description of +an issue. + +There is an [epic](https://gitlab.com/groups/gitlab-org/-/epics/2738) tracking the +addition of items such as issue assignees, labels, comments, user mapping, and much more. +These will be included in the future iterations of the GitLab Jira importer. + ## Prerequisites ### Permissions diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 585c45e35df..50272f0e007 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -242,7 +242,7 @@ field. For example: -```text +```plaintext machine example.gitlab.com login <gitlab_user_name> password <personal_access_token> diff --git a/doc/user/project/integrations/bamboo.md b/doc/user/project/integrations/bamboo.md index 8c7e6edbf38..db8f24fc1e1 100644 --- a/doc/user/project/integrations/bamboo.md +++ b/doc/user/project/integrations/bamboo.md @@ -27,7 +27,7 @@ need to be configured in a Bamboo build plan before GitLab can integrate. 1. In the left pane, select a build stage. If you have multiple build stages you want to select the last stage that contains the Git checkout task. 1. Select the 'Miscellaneous' tab. -1. Under 'Pattern Match Labelling' put `${bamboo.repository.revision.number}` +1. Under 'Pattern Match Labeling' put `${bamboo.repository.revision.number}` in the 'Labels' box. 1. Save diff --git a/doc/user/project/integrations/generic_alerts.md b/doc/user/project/integrations/generic_alerts.md index 1a000fd1c44..2234727dd82 100644 --- a/doc/user/project/integrations/generic_alerts.md +++ b/doc/user/project/integrations/generic_alerts.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Generic alerts integration > - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13203) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. @@ -26,12 +32,16 @@ You can customize the payload by sending the following parameters. All fields ar | Property | Type | Description | | -------- | ---- | ----------- | -| `title` | String | The title of the incident. If none is provided, then `New: Incident #N` will be used, where `#N` is the number of incident | +| `title` | String | The title of the incident. Required. | | `description` | String | A high-level summary of the problem. | | `start_time` | DateTime | The time of the incident. If none is provided, a timestamp of the issue will be used. | | `service` | String | The affected service. | | `monitoring_tool` | String | The name of the associated monitoring tool. | | `hosts` | String or Array | One or more hosts, as to where this incident occurred. | +| `severity` | String | The severity of the alert. Must be one of `critical`, `high`, `medium`, `low`, `info`, `unknown`. Default is `critical`. | + +TIP: **Payload size:** +Ensure your requests are smaller than the [payload application limits](../../../administration/instance_limits.md#generic-alert-json-payloads). Example request: diff --git a/doc/user/project/integrations/gitlab_slack_application.md b/doc/user/project/integrations/gitlab_slack_application.md index 1ef2f593621..49b6a3f6450 100644 --- a/doc/user/project/integrations/gitlab_slack_application.md +++ b/doc/user/project/integrations/gitlab_slack_application.md @@ -1,13 +1,14 @@ # GitLab Slack application **(FREE ONLY)** +> - Introduced in GitLab 9.4. +> - Distributed to Slack App Directory in GitLab 10.2. + NOTE: **Note:** The GitLab Slack application is only configurable for GitLab.com. It will **not** work for on-premises installations where you can configure the [Slack slash commands](slack_slash_commands.md) service instead. We're planning to make this configurable for all GitLab installations, but there's no ETA - see [#28164](https://gitlab.com/gitlab-org/gitlab/issues/28164). -It was first introduced in GitLab 9.4 and distributed to Slack App Directory in -GitLab 10.2. Slack provides a native application which you can enable via your project's integrations on GitLab.com. @@ -35,12 +36,30 @@ docs on [Adding an app to your team](https://slack.com/help/articles/202035138). To enable GitLab's service for your Slack team: -1. Go to your project's **Settings > Integration > Slack application** (only - visible on GitLab.com) -1. Click the "Add to Slack" button +1. Go to your project's **{settings}** **Settings > Integration > Slack application** (only + visible on GitLab.com). +1. Click **Add to Slack**. That's all! You can now start using the Slack slash commands. +## Create a project alias for Slack + +To create a project alias on GitLab.com for Slack integration: + +1. Go to your project's home page. +1. Navigate to **{settings}** **Settings > Integrations** (only visible on GitLab.com) +1. On the **Integrations** page, click **Slack application**. +1. The current **Project Alias**, if any, is displayed. To edit this value, + click **Edit**. +1. Enter your desired alias, and click **Save changes**. + +Some Slack commands require a project alias, and fail with the following error +if the project alias is incorrect or missing from the command: + +```plaintext +GitLab error: project or alias not found +``` + ## Usage After confirming the installation, you, and everyone else in your Slack team, diff --git a/doc/user/project/integrations/img/metrics_dashboard_annotations_ui_v13.0.png b/doc/user/project/integrations/img/metrics_dashboard_annotations_ui_v13.0.png Binary files differnew file mode 100644 index 00000000000..a042fbbcf4e --- /dev/null +++ b/doc/user/project/integrations/img/metrics_dashboard_annotations_ui_v13.0.png diff --git a/doc/user/project/integrations/img/panel_context_menu_v12_10.png b/doc/user/project/integrations/img/panel_context_menu_v12_10.png Binary files differdeleted file mode 100644 index 096262fea91..00000000000 --- a/doc/user/project/integrations/img/panel_context_menu_v12_10.png +++ /dev/null diff --git a/doc/user/project/integrations/img/panel_context_menu_v13_0.png b/doc/user/project/integrations/img/panel_context_menu_v13_0.png Binary files differnew file mode 100644 index 00000000000..2d7cb923981 --- /dev/null +++ b/doc/user/project/integrations/img/panel_context_menu_v13_0.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_select_v_13_0.png b/doc/user/project/integrations/img/prometheus_dashboard_select_v_13_0.png Binary files differnew file mode 100644 index 00000000000..2f0309ce664 --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_dashboard_select_v_13_0.png diff --git a/doc/user/project/integrations/img/toggle_metrics_user_starred_dashboard_v13_0.png b/doc/user/project/integrations/img/toggle_metrics_user_starred_dashboard_v13_0.png Binary files differnew file mode 100644 index 00000000000..59dc9ccfd30 --- /dev/null +++ b/doc/user/project/integrations/img/toggle_metrics_user_starred_dashboard_v13_0.png diff --git a/doc/user/project/integrations/img/webex_teams_configuration.png b/doc/user/project/integrations/img/webex_teams_configuration.png Binary files differnew file mode 100644 index 00000000000..66993e0887d --- /dev/null +++ b/doc/user/project/integrations/img/webex_teams_configuration.png diff --git a/doc/user/project/integrations/mattermost_slash_commands.md b/doc/user/project/integrations/mattermost_slash_commands.md index a23d8d7306d..6a202c9a130 100644 --- a/doc/user/project/integrations/mattermost_slash_commands.md +++ b/doc/user/project/integrations/mattermost_slash_commands.md @@ -17,21 +17,21 @@ If you have the Omnibus GitLab package installed, Mattermost is already bundled in it. All you have to do is configure it. Read more in the [Omnibus GitLab Mattermost documentation](https://docs.gitlab.com/omnibus/gitlab-mattermost/). -## Automated Configuration +## Automated configuration If Mattermost is installed on the same server as GitLab, the configuration process can be done for you by GitLab. Go to the Mattermost Slash Command service on your project and click the 'Add to Mattermost' button. -## Manual Configuration +## Manual configuration The configuration consists of two parts. First you need to enable the slash commands in Mattermost and then enable the service in GitLab. ### Step 1. Enable custom slash commands in Mattermost -This step is only required when using a source install, omnibus installs will be +This step is only required when using a source install, Omnibus installs will be preconfigured with the right settings. The first thing to do in Mattermost is to enable custom slash commands from @@ -145,6 +145,15 @@ trigger word followed by <kbd>help</kbd>. Example: `/gitlab help` The permissions to run the [available commands](#available-slash-commands) derive from the [permissions you have on the project](../../permissions.md#project-members-permissions). +## Troubleshooting + +If an event is not being triggered, confirm that the channel you're using is a public one, as +Mattermost webhooks do not have access to private channels. + +If a private channel is required, you can edit the webhook's channel in Mattermost and +select a private channel. It is not possible to use different channels for +different types of notifications - all events will be sent to the specified channel. + ## Further reading - [Mattermost slash commands documentation](https://docs.mattermost.com/developer/slash-commands.html) diff --git a/doc/user/project/integrations/overview.md b/doc/user/project/integrations/overview.md index b973abbe210..88668ab6c7d 100644 --- a/doc/user/project/integrations/overview.md +++ b/doc/user/project/integrations/overview.md @@ -47,7 +47,7 @@ Click on the service links to see further configuration instructions and details | [Microsoft teams](microsoft_teams.md) | Receive notifications for actions that happen on GitLab into a room on Microsoft Teams using Office 365 Connectors | No | | Packagist | Update your project on Packagist, the main Composer repository | Yes | | Pipelines emails | Email the pipeline status to a list of recipients | No | -| [Slack Notifications](slack.md) | Send GitLab events (e.g. issue created) to Slack as notifications | No | +| [Slack Notifications](slack.md) | Send GitLab events (for example, an issue was created) to Slack as notifications | No | | [Slack slash commands](slack_slash_commands.md) **(CORE ONLY)** | Use slash commands in Slack to control GitLab | No | | [GitLab Slack application](gitlab_slack_application.md) **(FREE ONLY)** | Use Slack's official application | No | | PivotalTracker | Project Management Software (Source Commits Endpoint) | No | @@ -55,6 +55,7 @@ Click on the service links to see further configuration instructions and details | Pushover | Pushover makes it easy to get real-time notifications on your Android device, iPhone, iPad, and Desktop | No | | [Redmine](redmine.md) | Redmine issue tracker | No | | [Unify Circuit](unify_circuit.md) | Receive events notifications in Unify Circuit | No | +| [Webex Teams](webex_teams.md) | Receive events notifications in Webex Teams | No | | [YouTrack](youtrack.md) | YouTrack issue tracker | No | ## Push hooks limit @@ -93,6 +94,15 @@ From this page, you can repeat delivery with the same data by clicking **Resend  +### Uninitialized repositories + +Some integrations fail with an error `Test Failed. Save Anyway` when you attempt to set them up on +uninitialized repositories. This is because the default service test uses push data to build the +payload for the test request, and it fails, because there are no push events for the project. + +To resolve this error, initialize the repository by pushing a test file to the project and set up +the integration again. + ## Contributing to integrations Because GitLab is open source we can ship with the code and tests for all @@ -100,9 +110,6 @@ plugins. This allows the community to keep the plugins up to date so that they always work in newer GitLab versions. For an overview of what integrations are available, please see the -[project_services source directory][projects-code]. +[project_services source directory](https://gitlab.com/gitlab-org/gitlab/tree/master/app/models/project_services). Contributions are welcome! - -[projects-code]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/models/project_services -[permissions]: ../../permissions.md diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index bbed14ea93f..6c40e5b9696 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Prometheus integration > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8935) in GitLab 9.0. @@ -66,6 +72,25 @@ It enables you to search as you type through all environments and select the one  +##### Select a dashboard + +The **dashboard** dropdown box above the dashboard displays the list of all dashboards available for the project. +It enables you to search as you type through all dashboards and select the one you're looking for. + + + +##### Mark a dashboard as favorite + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214582) in GitLab 13.0. + +When viewing a dashboard, click the empty **Star dashboard** **{star-o}** button to mark a +dashboard as a favorite. Starred dashboards display a solid star **{star}** button, +and appear at the top of the dashboard select list. + +To remove dashboard from the favorites list, click the solid **Unstar Dashboard** **{star}** button. + + + #### About managed Prometheus deployments Prometheus is deployed into the `gitlab-managed-apps` namespace, using the [official Helm chart](https://github.com/helm/charts/tree/master/stable/prometheus). Prometheus is only accessible within the cluster, with GitLab communicating through the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/). @@ -93,7 +118,7 @@ Integration with Prometheus requires the following: #### Getting started -Installing and configuring Prometheus to monitor applications is fairly straight forward. +Installing and configuring Prometheus to monitor applications is fairly straightforward. 1. [Install Prometheus](https://prometheus.io/docs/prometheus/latest/installation/) 1. Set up one of the [supported monitoring targets](prometheus_library/index.md) @@ -123,14 +148,32 @@ to integrate with. 1. Provide the domain name or IP address of your server, for example `http://thanos.example.com/` or `http://192.0.2.1/`. 1. Click **Save changes**. +### Precedence with multiple Prometheus configurations + +Although you can enable both a [manual configuration](#manual-configuration-of-prometheus) +and [auto configuration](#managed-prometheus-on-kubernetes) of Prometheus, only +one of them will be used: + +- If you have enabled a + [Prometheus manual configuration](#manual-configuration-of-prometheus) + and a [managed Prometheus on Kubernetes](#managed-prometheus-on-kubernetes), + the manual configuration takes precedence and is used to run queries from + [dashboards](#defining-custom-dashboards-per-project) and [custom metrics](#adding-custom-metrics). +- If you have managed Prometheus applications installed on Kubernetes clusters + at **different** levels (project, group, instance), the order of precedence is described in + [Cluster precedence](../../instance/clusters/index.md#cluster-precedence). +- If you have managed Prometheus applications installed on multiple Kubernetes + clusters at the **same** level, the Prometheus application of a cluster with a + matching [environment scope](../../../ci/environments/index.md#scoping-environments-with-specs) is used. + ## Monitoring CI/CD Environments Once configured, GitLab will attempt to retrieve performance metrics for any environment which has had a successful deployment. -GitLab will automatically scan the Prometheus server for metrics from known servers like Kubernetes and NGINX, and attempt to identify individual environment. The supported metrics and scan process is detailed in our [Prometheus Metrics Library documentation](prometheus_library/index.md). +GitLab will automatically scan the Prometheus server for metrics from known servers like Kubernetes and NGINX, and attempt to identify individual environments. The supported metrics and scan process is detailed in our [Prometheus Metrics Library documentation](prometheus_library/index.md). -You can view the performance dashboard for an environment by [clicking on the monitoring button](../../../ci/environments.md#monitoring-environments). +You can view the performance dashboard for an environment by [clicking on the monitoring button](../../../ci/environments/index.md#monitoring-environments). ### Adding custom metrics @@ -152,10 +195,12 @@ A few fields are required: - **Y-axis label**: Y axis title to display on the dashboard. - **Unit label**: Query units, for example `req / sec`. Shown next to the value. -Multiple metrics can be displayed on the same chart if the fields **Name**, **Type**, and **Y-axis label** match between metrics. For example, a metric with **Name** `Requests Rate`, **Type** `Business`, and **Y-axis label** `rec / sec` would display on the same chart as a second metric with the same values. A **Legend label** is suggested if this feature used. +Multiple metrics can be displayed on the same chart if the fields **Name**, **Type**, and **Y-axis label** match between metrics. For example, a metric with **Name** `Requests Rate`, **Type** `Business`, and **Y-axis label** `rec / sec` would display on the same chart as a second metric with the same values. A **Legend label** is suggested if this feature is used. #### Query Variables +##### Predefined variables + GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) in the Prometheus query. This is particularly useful for identifying a specific environment, for example with `ci_environment_slug`. The supported variables are: - `ci_environment_slug` @@ -168,10 +213,34 @@ GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) NOTE: **Note:** Variables for Prometheus queries must be lowercase. -There are 2 methods to specify a variable in a query or dashboard: +##### User-defined variables + +[Variables can be defined](#templating-templating-properties) in a custom dashboard YAML file. + +##### Using variables + +Variables can be specified using double curly braces, such as `"{{ci_environment_slug}}"` ([added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20793) in GitLab 12.7). + +Support for the `"%{ci_environment_slug}"` format was +[removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31581) in GitLab 13.0. +Queries that continue to use the old format will show no data. + +#### Query Variables from URL -1. Variables can be specified using the [Liquid template format](https://shopify.dev/docs/liquid/reference/basics), for example `{{ci_environment_slug}}` ([added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20793) in GitLab 12.6). -1. You can also enclose it in quotation marks with curly braces with a leading percent, for example `"%{ci_environment_slug}"`. This method is deprecated though and support will be [removed in the next major release](https://gitlab.com/gitlab-org/gitlab/issues/37990). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214500) in GitLab 13.0. + +GitLab supports setting custom variables through URL parameters. Surround the variable +name with double curly braces (`{{example}}`) to interpolate the variable in a query: + +```plaintext +avg(sum(container_memory_usage_bytes{container_name!="{{pod}}"}) by (job)) without (job) /1024/1024/1024' +``` + +The URL for this query would be: + +```plaintext +http://gitlab.com/<user>/<project>/-/environments/<environment_id>/metrics?dashboard=.gitlab%2Fdashboards%2Fcustom.yml&pod=POD +``` #### Editing additional metrics from the dashboard @@ -261,19 +330,29 @@ If you select another branch, this branch should be merged to your **default** b Dashboards have several components: -- Panel groups, which comprise of panels. +- Templating variables. +- Panel groups, which consist of panels. - Panels, which support one or more metrics. The following tables outline the details of expected properties. -**Dashboard properties:** +##### **Dashboard (top-level) properties** | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | | `dashboard` | string | yes | Heading for the dashboard. Only one dashboard should be defined per file. | | `panel_groups` | array | yes | The panel groups which should be on the dashboard. | +| `templating` | Hash | no | Top level key under which templating related options can be added. | + +##### **Templating (`templating`) properties** + +| Property | Type | Required | Description | +| -------- | ---- | -------- | ----------- | +| `variables` | Hash | no | Variables can be defined here. | -**Panel group (`panel_groups`) properties:** +Read the documentation on [templating](#templating-variables-for-metrics-dashboards). + +##### **Panel group (`panel_groups`) properties** | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | @@ -281,7 +360,7 @@ The following tables outline the details of expected properties. | `priority` | number | optional, defaults to order in file | Order to appear on the dashboard. Higher number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | | `panels` | array | required | The panels which should be in the panel group. | -**Panel (`panels`) properties:** +##### **Panel (`panels`) properties** | Property | Type | Required | Description | | ------ | ------ | ------ | ------- | @@ -293,19 +372,19 @@ The following tables outline the details of expected properties. | `weight` | number | no, defaults to order in file | Order to appear within the grouping. Lower number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | | `metrics` | array | yes | The metrics which should be displayed in the panel. Any number of metrics can be displayed when `type` is `area-chart` or `line-chart`, whereas only 3 can be displayed when `type` is `anomaly-chart`. | -**Axis (`panels[].y_axis`) properties:** +##### **Axis (`panels[].y_axis`) properties** -| Property | Type | Required | Description | -| ----------- | ------ | ------------------------- | -------------------------------------------------------------------- | -| `name` | string | no, but highly encouraged | Y-Axis label for the panel, it will replace `y_label` if set. | -| `format` | string | no, defaults to `number` | Unit format used. See the [full list of units](prometheus_units.md). | -| `precision` | number | no, defaults to `2` | Number of decimals to display in the number. | +| Property | Type | Required | Description | +| ----------- | ------ | ----------------------------- | -------------------------------------------------------------------- | +| `name` | string | no, but highly encouraged | Y-Axis label for the panel. Replaces `y_label` if set. | +| `format` | string | no, defaults to `engineering` | Unit format used. See the [full list of units](prometheus_units.md). | +| `precision` | number | no, defaults to `2` | Number of decimal places to display in the number. | | -**Metrics (`metrics`) properties:** +##### **Metrics (`metrics`) properties** | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | -| `id` | string | no | Used for associating dashboard metrics with database records. Must be unique across dashboard configuration files. Required for [alerting](#setting-up-alerts-for-prometheus-metrics) (support not yet enabled, see [relevant issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/60319)). | +| `id` | string | no | Used for associating dashboard metrics with database records. Must be unique across dashboard configuration files. Required for [alerting](#setting-up-alerts-for-prometheus-metrics) (support not yet enabled, see [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/27980)). | | `unit` | string | yes | Defines the unit of the query's return data. | | `label` | string | no, but highly encouraged | Defines the legend-label for the query. Should be unique within the panel's metrics. Can contain time series labels as interpolated variables. | | `query` | string | yes if `query_range` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | @@ -610,21 +689,122 @@ Note the following properties:  +### Templating variables for metrics dashboards + +Templating variables can be used to make your metrics dashboard more versatile. + +#### Templating variable types + +`templating` is a top-level key in the +[dashboard YAML](#dashboard-top-level-properties). +Define your variables in the `variables` key, under `templating`. The value of +the `variables` key should be a hash, and each key under `variables` +defines a templating variable on the dashboard. + +A variable can be used in a Prometheus query in the same dashboard using the syntax +described [here](#using-variables). + +##### `text` variable type + +CAUTION: **Warning:** +This variable type is an _alpha_ feature, and is subject to change at any time +without prior notice! + +For each `text` variable defined in the dashboard YAML, there will be a free text +box on the dashboard UI, allowing you to enter a value for each variable. + +The `text` variable type supports a simple and a full syntax. + +###### Simple syntax + +This example creates a variable called `variable1`, with a default value +of `default value`: + +```yaml +templating: + variables: + variable1: 'default value' # `text` type variable with `default value` as its default. +``` + +###### Full syntax + +This example creates a variable called `variable1`, with a default value of `default`. +The label for the text box on the UI will be the value of the `label` key: + +```yaml +templating: + variables: + variable1: # The variable name that can be used in queries. + label: 'Variable 1' # (Optional) label that will appear in the UI for this text box. + type: text + options: + default_value: 'default' # (Optional) default value. +``` + +##### `custom` variable type + +CAUTION: **Warning:** +This variable type is an _alpha_ feature, and is subject to change at any time +without prior notice! + +Each `custom` variable defined in the dashboard YAML creates a dropdown +selector on the dashboard UI, allowing you to select a value for each variable. + +The `custom` variable type supports a simple and a full syntax. + +###### Simple syntax + +This example creates a variable called `variable1`, with a default value of `value1`. +The dashboard UI will display a dropdown with `value1`, `value2` and `value3` +as the choices. + +```yaml +templating: + variables: + variable1: ['value1', 'value2', 'value3'] +``` + +###### Full syntax + +This example creates a variable called `variable1`, with a default value of `var1_option_2`. +The label for the text box on the UI will be the value of the `label` key. +The dashboard UI will display a dropdown with `Option 1` and `Option 2` +as the choices. + +If you select `Option 1` from the dropdown, the variable will be replaced with `value option 1`. +Similarly, if you select `Option 2`, the variable will be replaced with `value_option_2`: + +```yaml +templating: + variables: + variable1: # The variable name that can be used in queries. + label: 'Variable 1' # (Optional) label that will appear in the UI for this dropdown. + type: custom + options: + values: + - value: 'value option 1' # The value that will replace the variable in queries. + text: 'Option 1' # (Optional) Text that will appear in the UI dropdown. + - value: 'value_option_2' + text: 'Option 2' + default: true # (Optional) This option should be the default value of this variable. +``` + ### View and edit the source file of a custom dashboard > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34779) in GitLab 12.5. When viewing a custom dashboard of a project, you can view the original -`.yml` file by clicking on **Edit dashboard** button. +`.yml` file by clicking on the **Edit dashboard** button. ### Chart Context Menu From each of the panels in the dashboard, you can access the context menu by clicking the **{ellipsis_v}** **More actions** dropdown box above the upper right corner of the panel to take actions related to the chart's data. - + The options are: +- [Expand panel](#expand-panel) - [View logs](#view-logs-ultimate) - [Download CSV](#downloading-data-as-csv) - [Copy link to chart](#embedding-gitlab-managed-kubernetes-metrics) @@ -632,7 +812,8 @@ The options are: ### Dashboard Annotations -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/215224) in GitLab 13.0. You can use **Metrics Dashboard Annotations** to mark any important events on every metrics dashboard by adding annotations to it. While viewing a dashboard, @@ -644,6 +825,18 @@ its description. You can create annotations by making requests to the [Metrics dashboard annotations API](../../../api/metrics_dashboard_annotations.md) + + +### Expand panel + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0. + +To view a larger version of a visualization, expand the panel by clicking the +**{ellipsis_v}** **More actions** icon and selecting **Expand panel**. + +To return to the metrics dashboard, click the **Back** button in your +browser, or pressing the <kbd>Esc</kbd> key. + ### View Logs **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/122013) in GitLab 12.8. @@ -708,14 +901,14 @@ receivers: ... ``` -In order for GitLab to associate your alerts with an [environment](../../../ci/environments.md), you need to configure a `gitlab_environment_name` label on the alerts you set up in Prometheus. The value of this should match the name of your Environment in GitLab. +In order for GitLab to associate your alerts with an [environment](../../../ci/environments/index.md), you need to configure a `gitlab_environment_name` label on the alerts you set up in Prometheus. The value of this should match the name of your Environment in GitLab. ### Taking action on incidents **(ULTIMATE)** >- [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4925) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. >- [From GitLab Ultimate 12.5](https://gitlab.com/gitlab-org/gitlab/issues/13401), when GitLab receives a recovery alert, it will automatically close the associated issue. -Alerts can be used to trigger actions, like open an issue automatically (enabled by default since `12.1`). To configure the actions: +Alerts can be used to trigger actions, like opening an issue automatically (enabled by default since `12.1`). To configure the actions: 1. Navigate to your project's **Settings > Operations > Incidents**. 1. Enable the option to create issues. @@ -734,7 +927,7 @@ Once enabled, an issue will be opened automatically when an alert is triggered w - Optional list of attached annotations extracted from `annotations/*` - Alert [GFM](../../markdown.md): GitLab Flavored Markdown from `annotations/gitlab_incident_markdown` -When GitLab receives a **Recovery Alert**, it will automatically close the associated issue. This action will be recorded as a system message on the issue indicated that it was closed automatically by the GitLab Alert bot. +When GitLab receives a **Recovery Alert**, it will automatically close the associated issue. This action will be recorded as a system message on the issue indicating that it was closed automatically by the GitLab Alert bot. To further customize the issue, you can add labels, mentions, or any other supported [quick action](../quick_actions.md) in the selected issue template, which will apply to all incidents. To limit quick actions or other information to only specific types of alerts, use the `annotations/gitlab_incident_markdown` field. @@ -812,6 +1005,8 @@ Metric charts may also be hidden:  +You can open the link directly into your browser for a [detailed view of the data](#expand-panel). + ### Embedding metrics in issue templates It is also possible to embed either the default dashboard metrics or individual metrics in issue templates. For charts to render side-by-side, links to the entire metrics dashboard or individual metrics should be separated by either a comma or a space. @@ -907,12 +1102,20 @@ Prerequisites for embedding from a Grafana instance: 1. In the upper-left corner of the page, select a specific value for each variable required for the queries in the chart.  1. In Grafana, click on a panel's title, then click **Share** to open the panel's sharing dialog to the **Link** tab. If you click the _dashboard's_ share panel instead, GitLab will attempt to embed the first supported panel on the dashboard (if available). -1. If your Prometheus queries use Grafana's custom template variables, ensure that "Template variables" option is toggled to **On**. Of Grafana global template variables, only `$__interval`, `$__from`, and `$__to` are currently supported. Toggle **On** the "Current time range" option to specify the time range of the chart. Otherwise, the default range will be the last 8 hours. +1. If your Prometheus queries use Grafana's custom template variables, ensure that the "Template variables" option is toggled to **On**. Of Grafana global template variables, only `$__interval`, `$__from`, and `$__to` are currently supported. Toggle **On** the "Current time range" option to specify the time range of the chart. Otherwise, the default range will be the last 8 hours.  1. Click **Copy** to copy the URL to the clipboard. 1. In GitLab, paste the URL into a Markdown field and save. The chart will take a few moments to render.  +## Metrics dashboard visibility + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201924) in GitLab 13.0. + +You can set the visibility of the metrics dashboard to **Only Project Members** +or **Everyone With Access**. When set to **Everyone with Access**, the metrics +dashboard is visible to authenticated and non-authenticated users. + ## Troubleshooting When troubleshooting issues with a managed Prometheus app, it is often useful to diff --git a/doc/user/project/integrations/prometheus_library/cloudwatch.md b/doc/user/project/integrations/prometheus_library/cloudwatch.md index 143130aebea..911493cdae9 100644 --- a/doc/user/project/integrations/prometheus_library/cloudwatch.md +++ b/doc/user/project/integrations/prometheus_library/cloudwatch.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring AWS Resources > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12621) in GitLab 9.4 diff --git a/doc/user/project/integrations/prometheus_library/haproxy.md b/doc/user/project/integrations/prometheus_library/haproxy.md index fa3590af8cf..712805b75f2 100644 --- a/doc/user/project/integrations/prometheus_library/haproxy.md +++ b/doc/user/project/integrations/prometheus_library/haproxy.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring HAProxy > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12621) in GitLab 9.4 diff --git a/doc/user/project/integrations/prometheus_library/index.md b/doc/user/project/integrations/prometheus_library/index.md index c2b3676b23f..6f2c2477eee 100644 --- a/doc/user/project/integrations/prometheus_library/index.md +++ b/doc/user/project/integrations/prometheus_library/index.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Prometheus Metrics library > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8935) in GitLab 9.0. diff --git a/doc/user/project/integrations/prometheus_library/kubernetes.md b/doc/user/project/integrations/prometheus_library/kubernetes.md index ca1555c793b..29efe08e53d 100644 --- a/doc/user/project/integrations/prometheus_library/kubernetes.md +++ b/doc/user/project/integrations/prometheus_library/kubernetes.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring Kubernetes > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8935) in GitLab 9.0. diff --git a/doc/user/project/integrations/prometheus_library/nginx.md b/doc/user/project/integrations/prometheus_library/nginx.md index d5f078f3ddf..eda6f64ccac 100644 --- a/doc/user/project/integrations/prometheus_library/nginx.md +++ b/doc/user/project/integrations/prometheus_library/nginx.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring NGINX > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12621) in GitLab 9.4 diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress.md b/doc/user/project/integrations/prometheus_library/nginx_ingress.md index 62f8c08e298..b2bc217e8bf 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring NGINX Ingress Controller > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22133) in GitLab 11.7. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md index af3b725deb6..6ba0a7610f6 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring NGINX Ingress Controller with VTS metrics > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13438) in GitLab 9.5. diff --git a/doc/user/project/integrations/prometheus_units.md b/doc/user/project/integrations/prometheus_units.md index 9df9f52ceb1..691d20e5de2 100644 --- a/doc/user/project/integrations/prometheus_units.md +++ b/doc/user/project/integrations/prometheus_units.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Unit formats reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201999) in GitLab 12.9. @@ -5,19 +11,78 @@ You can select units to format your charts by adding `format` to your [axis configuration](prometheus.md#dashboard-yaml-properties). +## Internationalization and localization + +Currently, your [internationalization and localization options](https://en.wikipedia.org/wiki/Internationalization_and_localization) for number formatting are dependent on the system you are using i.e. your OS or browser. + +## Engineering Notation + +For generic or default data, numbers are formatted according to the current locale in [engineering notation](https://en.wikipedia.org/wiki/Engineering_notation). + +While an [engineering notation exists for the web](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NumberFormat), GitLab uses a version based off the [scientific notation](https://en.wikipedia.org/wiki/Scientific_notation). GitLab formatting acts in accordance with SI prefixes. For example, using GitLab notation, `1500.00` becomes `1.5k` instead of `1.5E3`. Keep this distinction in mind when using the engineering notation for your metrics. + +Formats: `engineering` + +SI prefixes: + +| Name | Symbol | Value | +| ---------- | ------- | -------------------------- | +| `yotta` | Y | 1000000000000000000000000 | +| `zetta` | Z | 1000000000000000000000 | +| `exa` | E | 1000000000000000000 | +| `peta` | P | 1000000000000000 | +| `tera` | T | 1000000000000 | +| `giga` | G | 1000000000 | +| `mega` | M | 1000000 | +| `kilo` | k | 1000 | +| `milli` | m | 0.001 | +| `micro` | μ | 0.000001 | +| `nano` | n | 0.000000001 | +| `pico` | p | 0.000000000001 | +| `femto` | f | 0.000000000000001 | +| `atto` | a | 0.000000000000000001 | +| `zepto` | z | 0.000000000000000000001 | +| `yocto` | y | 0.000000000000000000000001 | + +**Examples:** + +| Data | Displayed | +| --------------------------------- | --------- | +| `0.000000000000000000000008` | 8y | +| `0.000000000000000000008` | 8z | +| `0.000000000000000008` | 8a | +| `0.000000000000008` | 8f | +| `0.000000000008` | 8p | +| `0.000000008` | 8n | +| `0.000008` | 8μ | +| `0.008` | 8m | +| `10` | 10 | +| `1080` | 1.08k | +| `18000` | 18k | +| `18888` | 18.9k | +| `188888` | 189k | +| `18888888` | 18.9M | +| `1888888888` | 1.89G | +| `1888888888888` | 1.89T | +| `1888888888888888` | 1.89P | +| `1888888888888888888` | 1.89E | +| `1888888888888888888888` | 1.89Z | +| `1888888888888888888888888` | 1.89Y | +| `1888888888888888888888888888` | 1.89e+27 | + ## Numbers -For generic data, numbers are formatted according to the current locale. +For number data, numbers are formatted according to the current locale. Formats: `number` **Examples:** -| Data | Displayed | -| --------- | --------- | -| `10` | 1 | -| `1000` | 1,000 | -| `1000000` | 1,000,000 | +| Data | Displayed | +| ---------- | --------- | +| `10` | 1 | +| `1000` | 1,000 | +| `1000000` | 1,000,000 | ## Percentage diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index ba2a8f55d84..419340c14ef 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -41,7 +41,7 @@ an error message and keep troubleshooting from there. You may see an entry similar to the following in your Sidekiq log: -```text +```plaintext 2019-01-10_13:22:08.42572 2019-01-10T13:22:08.425Z 6877 TID-abcdefg ProjectServiceWorker JID-3bade5fb3dd47a85db6d78c5 ERROR: {:class=>"ProjectServiceWorker", :service_class=>"SlackService", :message=>"SSL_connect returned=1 errno=0 state=error: certificate verify failed"} ``` diff --git a/doc/user/project/integrations/webex_teams.md b/doc/user/project/integrations/webex_teams.md new file mode 100644 index 00000000000..a6e688887b6 --- /dev/null +++ b/doc/user/project/integrations/webex_teams.md @@ -0,0 +1,24 @@ +# Webex Teams service + +You can configure GitLab to send notifications to a Webex Teams space. + +## Create a webhook for the space + +1. Go to the [Incoming Webooks app page](https://apphub.webex.com/teams/applications/incoming-webhooks-cisco-systems). +1. Click **Connect** and log in to Webex Teams, if required. +1. Enter a name for the webhook and select the space that will receive the notifications. +1. Click **ADD**. +1. Copy the **Webhook URL**. + +## Configure settings in GitLab + +Once you have a webhook URL for your Webex Teams space, you can configure GitLab to send notifications. + +1. Navigate to **Project > Settings > Integrations**. +1. Select the **Webex Teams** integration. +1. Ensure that the **Active** toggle is enabled. +1. Select the checkboxes corresponding to the GitLab events you want to receive in Webex Teams. +1. Paste the **Webhook** URL for the Webex Teams space. +1. Configure the remaining options and then click **Test settings and save changes**. + +The Webex Teams space will begin to receive all applicable GitLab events. diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 6dd2fd3b61b..89e84dda0e8 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -1294,7 +1294,7 @@ X-Gitlab-Event: Job Hook } ``` -Note that `commit.id` is the id of the pipeline, not the id of the commit. +Note that `commit.id` is the ID of the pipeline, not the ID of the commit. ## Image URL rewriting diff --git a/doc/user/project/integrations/youtrack.md b/doc/user/project/integrations/youtrack.md index a72eaaa9ff1..119a53f5c35 100644 --- a/doc/user/project/integrations/youtrack.md +++ b/doc/user/project/integrations/youtrack.md @@ -14,8 +14,8 @@ To enable YouTrack integration in a project: | Field | Description | |:----------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Description** | Name for the issue tracker (to differentiate between instances, for example). | - | **Project url** | URL to the project in YouTrack which is being linked to this GitLab project. | - | **Issues url** | URL to the issue in YouTrack project that is linked to this GitLab project. Note that the **Issues url** requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. | + | **Project URL** | URL to the project in YouTrack which is being linked to this GitLab project. | + | **Issues URL** | URL to the issue in YouTrack project that is linked to this GitLab project. Note that the **Issues URL** requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. | 1. Click the **Save changes** button. diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 5bc71337e44..9903a711987 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -9,220 +9,271 @@ organize, and visualize a workflow for a feature or product release. It can be used as a [Kanban](https://en.wikipedia.org/wiki/Kanban_(development)) or a [Scrum](https://en.wikipedia.org/wiki/Scrum_(software_development)) board. -It provides perfect pairing between issue tracking and project management, +It pairs issue tracking and project management, keeping everything in the same place, so that you don't need to jump between different platforms to organize your workflow. -With GitLab Issue Boards, you organize your issues in lists that correspond to +With issue boards, you organize your issues in lists that correspond to their assigned labels, visualizing issues designed as cards throughout those lists. -You define your process and GitLab organizes it for you. You add your labels +You define your process, and GitLab organizes it for you. You add your labels then create the corresponding list to pull in your existing issues. When you're ready, you can drag and drop your issue cards from one step to the next. - + -**<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video presentation](https://youtu.be/UWsJ8tkHAa8) of -Issue Boards** (version introduced in GitLab 8.11 - August 2016). +the Issue Board feature (introduced in GitLab 8.11 - August 2016). -### Advanced features of Issue Boards +### Advanced features of issue boards - Create multiple issue boards per project. - Create multiple issue boards per group. **(PREMIUM)** - Add lists for [assignees](#assignee-lists-premium) and [milestones](#milestone-lists-premium). **(PREMIUM)** -Check all the [advanced features of Issue Boards](#gitlab-enterprise-features-for-issue-boards) -below. +Check all the [GitLab Enterprise features for issue boards](#gitlab-enterprise-features-for-issue-boards). - + + +--- ## How it works -The Issue Board builds on GitLab's existing +The Issue Board feature builds on GitLab's existing [issue tracking functionality](issues/index.md#issues-list) and -leverages the power of [labels](labels.md) by utilizing them as lists of the scrum board. +[labels](labels.md) by using them as lists of the Scrum board. -With the Issue Board you can have a different view of your issues while +With issue boards you can have a different view of your issues while maintaining the same filtering and sorting abilities you see across the -issue tracker. An Issue Board is based on its project's label structure, therefore, it +issue tracker. An issue board is based on its project's label structure, so it applies the same descriptive labels to indicate placement on the board, keeping consistency throughout the entire development lifecycle. -An Issue Board shows you what issues your team is working on, who is assigned to each, +An issue board shows you what issues your team is working on, who is assigned to each, and where in the workflow those issues are. You create issues, host code, perform reviews, build, test, -and deploy from one single platform. Issue Boards help you to visualize -and manage the entire process _in_ GitLab. +and deploy from one single platform. Issue boards help you to visualize +and manage the entire process in GitLab. -With [Multiple Issue Boards](#use-cases-for-multiple-issue-boards), +With [multiple issue boards](#use-cases-for-multiple-issue-boards), you go even further, as you can not only keep yourself and your project -organized from a broader perspective with one Issue Board per project, +organized from a broader perspective with one issue board per project, but also allow your team members to organize their own workflow by creating -multiple Issue Boards within the same project. +multiple issue boards within the same project. ## Use cases -There are many ways to use GitLab Issue Boards tailored to your own preferred workflow. -Here are some common use cases for Issue Boards. +There are many ways to use GitLab issue boards tailored to your own preferred workflow. +Here are some common use cases for issue boards. -### Use cases for a single Issue Board +### Use cases for a single issue board -GitLab Workflow allows you to discuss proposals in issues, categorize them -with labels, and from there organize and prioritize them with Issue Boards. +With the GitLab Workflow you can discuss proposals in issues, categorize them +with labels, and from there, organize and prioritize them with issue boards. For example, let's consider this simplified development workflow: -1. You have a repository hosting your app's codebase - and your team actively contributing to code -1. Your **backend** team starts working a new - implementation, gathers feedback and approval, and pass it over to **frontend** -1. When frontend is complete, the new feature is deployed to **staging** to be tested -1. When successful, it is deployed to **production** +1. You have a repository that hosts your application's codebase, and your team actively contributes code. +1. Your **backend** team starts working on a new implementation, gathers feedback and approval, and + passes it over to the **frontend** team. +1. When frontend is complete, the new feature is deployed to a **staging** environment to be tested. +1. When successful, it's deployed to **production**. -If we have the labels "**backend**", "**frontend**", "**staging**", and -"**production**", and an Issue Board with a list for each, we can: +If you have the labels "**backend**", "**frontend**", "**staging**", and +"**production**", and an issue board with a list for each, you can: -- Visualize the entire flow of implementations since the - beginning of the development life cycle until deployed to production -- Prioritize the issues in a list by moving them vertically -- Move issues between lists to organize them according to the labels you've set -- Add multiple issues to lists in the board by selecting one or more existing issues +- Visualize the entire flow of implementations since the beginning of the development life cycle + until deployed to production. +- Prioritize the issues in a list by moving them vertically. +- Move issues between lists to organize them according to the labels you've set. +- Add multiple issues to lists in the board by selecting one or more existing issues.  -### Use cases for Multiple Issue Boards +--- + +### Use cases for multiple issue boards -With [Multiple Issue Boards](#multiple-issue-boards), +With [multiple issue boards](#multiple-issue-boards), each team can have their own board to organize their workflow individually. #### Scrum team -With Multiple Issue Boards, each team has one board. Now you can move issues through each +With multiple issue boards, each team has one board. Now you can move issues through each part of the process. For instance: **To Do**, **Doing**, and **Done**. #### Organization of topics -Create lists to order things by topic and quickly change them between topics or groups, -such as between **UX**, **Frontend**, and **Backend**. The changes will be reflected across boards, -as changing lists will update the label accordingly. +Create lists to order issues by topic and quickly change them between topics or groups, +such as between **UX**, **Frontend**, and **Backend**. The changes are reflected across boards, +as changing lists updates the labels on each issue accordingly. #### Advanced team handover -For example, suppose we have a UX team with an Issue Board that contains: +For example, suppose we have a UX team with an issue board that contains: - **To Do** - **Doing** - **Frontend** -When done with something, they move the card to **Frontend**. The Frontend team's board looks like: +When finished with something, they move the card to **Frontend**. The Frontend team's board looks like: - **Frontend** - **Doing** - **Done** -Cards finished by the UX team will automatically appear in the **Frontend** column when they're ready for them. +Cards finished by the UX team automatically appear in the **Frontend** column when they are ready +for them. NOTE: **Note:** For a broader use case, please see the blog post [GitLab Workflow, an Overview](https://about.gitlab.com/blog/2016/10/25/gitlab-workflow-an-overview/#gitlab-workflow-use-case-scenario). For a real use case example, you can read why -[Codepen decided to adopt Issue Boards](https://about.gitlab.com/blog/2017/01/27/codepen-welcome-to-gitlab/#project-management-everything-in-one-place) +[Codepen decided to adopt issue boards](https://about.gitlab.com/blog/2017/01/27/codepen-welcome-to-gitlab/#project-management-everything-in-one-place) to improve their workflow with multiple boards. #### Quick assignments -Create lists for each of your team members and quickly drag-and-drop issues onto each team member. +Create lists for each of your team members and quickly drag and drop issues onto each team member's +list. + +## Issue board terminology + +An **issue board** represents a unique view of your issues. It can have multiple lists with each +list consisting of issues represented by cards. + +A **list** is a column on the issue board that displays issues matching certain attributes. +In addition to the default "Open" and "Closed" lists, each additional list shows issues matching +your chosen label, assignee, or milestone. On the top of each list you can see the number of issues +that belong to it. Types of lists include: + +- **Open** (default): all open issues that do not belong to one of the other lists. + Always appears as the leftmost list. +- **Closed** (default): all closed issues. Always appears as the rightmost list. +- **Label list**: all open issues for a label. +- [**Assignee list**](#assignee-lists-premium): all open issues assigned to a user. +- [**Milestone list**](#milestone-lists-premium): all open issues for a milestone. -## Issue Board terminology +A **Card** is a box on a list, and it represents an issue. You can drag cards from one list to +another to change their label, assignee, or milestone. The information you can see on a +card includes: -- **Issue Board** - Each board represents a unique view for your issues. It can have multiple lists with each list consisting of issues represented by cards. -- **List** - A column on the issue board that displays issues matching certain attributes. In addition to the default lists of 'Open' and 'Closed' issue, each additional list will show issues matching your chosen label or assignee. On the top of that list you can see the number of issues that belong to it. - - **Label list**: a list based on a label. It shows all opened issues with that label. - - **Assignee list**: a list which includes all issues assigned to a user. - - **Open** (default): shows all open issues that do not belong to one of the other lists. Always appears as the leftmost list. - - **Closed** (default): shows all closed issues. Always appears as the rightmost list. -- **Card** - A box in the list that represents an individual issue. The information you can see on a card consists of the issue number, the issue title, the assignee, and the labels associated with the issue. You can drag cards from one list to another to change their label or assignee from that of the source list to that of the destination list. +- Issue title +- Associated labels +- Issue number +- Assignee ## Permissions -[Reporters and up](../permissions.md) can use all the functionality of the -Issue Board to create or delete lists, and drag issues from one list to another. +Users with the [Reporter and higher roles](../permissions.md) can use all the functionality of the +Issue Board feature to create or delete lists and drag issues from one list to another. -## GitLab Enterprise features for Issue Boards +## GitLab Enterprise features for issue boards -GitLab Issue Boards are available on GitLab Core and GitLab.com Free, but some -advanced functionalities are only present in higher tiers: GitLab.com Bronze, -Silver, or Gold, or GitLab self-managed Starter, Premium, and Ultimate, as described -in the following sections. +GitLab issue boards are available on GitLab Core and GitLab.com Free tiers, but some +advanced functionality is present in [higher tiers only](https://about.gitlab.com/pricing/). -For a collection of [features per tier](#summary-of-features-per-tier), check the summary below. +### Summary of features per tier + +Different issue board features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), +as shown in the following table: -### Multiple Issue Boards +| Tier | Number of Project issue boards | Number of Group issue boards | Configurable issue boards | Assignee lists | +|------------------|--------------------------------|------------------------------|---------------------------|----------------| +| Core / Free | Multiple | 1 | No | No | +| Starter / Bronze | Multiple | 1 | Yes | No | +| Premium / Silver | Multiple | Multiple | Yes | Yes | +| Ultimate / Gold | Multiple | Multiple | Yes | Yes | -> - Multiple Issue Boards per project [moved](https://gitlab.com/gitlab-org/gitlab-foss/issues/53811) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 12.1. -> - Multiple Issue Boards per group is available in [GitLab Premium Edition](https://about.gitlab.com/pricing/). +### Multiple issue boards -Multiple Issue Boards, as the name suggests, allow for more than one Issue Board -for a given project or group. This is great for large projects with more than one team -or in situations where a repository is used to host the code of multiple -products. +> - [Introduced](https://about.gitlab.com/releases/2016/10/22/gitlab-8-13-released/) in GitLab 8.13. +> - Multiple issue boards per project [moved](https://gitlab.com/gitlab-org/gitlab-foss/issues/53811) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 12.1. +> - Multiple issue boards per group are available in [GitLab Premium](https://about.gitlab.com/pricing/). + +Multiple issue boards allow for more than one issue board for a given project or group. +This is great for large projects with more than one team or in situations where a repository is used +to host the code of multiple products. -Clicking on the current board name in the upper left corner will reveal a -menu from where you can create another Issue Board or delete the existing one. Using the search box at the top of the menu, you can filter the listed boards. -When you have 10 or more boards available, a "Recent" section is also shown in the menu. -These are shortcuts to your last 4 visited boards. +When you have ten or more boards available, a **Recent** section is also shown in the menu, with +shortcuts to your last four visited boards. - + When you're revisiting an issue board in a project or group with multiple boards, -GitLab will automatically load the last board you visited. +GitLab automatically loads the last board you visited. + +#### Create an issue board -### Configurable Issue Boards **(STARTER)** +To create a new issue board: -> Introduced in [GitLab Starter Edition 10.2](https://about.gitlab.com/releases/2017/11/22/gitlab-10-2-released/#issue-boards-configuration). +1. Click the dropdown with the current board name in the upper left corner of the Issue Boards page. +1. Click **Create new board**. +1. Enter the new board's name and select its scope: milestone, labels, assignee, or weight. -An Issue Board can be associated with a GitLab [Milestone](milestones/index.md#milestones), +#### Delete an issue board + +To delete the currently active issue board: + +1. Click the dropdown with the current board name in the upper left corner of the Issue Boards page. +1. Click **Delete board**. +1. Click **Delete** to confirm. + +### Configurable issue boards **(STARTER)** + +> [Introduced](https://about.gitlab.com/releases/2017/11/22/gitlab-10-2-released/#issue-boards-configuration) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.2. + +An issue board can be associated with a GitLab [Milestone](milestones/index.md#milestones), [Labels](labels.md), Assignee and Weight which will automatically filter the Board issues according to these fields. This allows you to create unique boards according to your team's need.  -You can define the scope of your board when creating it or by clicking on the "Edit board" button. -Once a milestone, assignee or weight is assigned to an Issue Board, you will no longer be able to +You can define the scope of your board when creating it or by clicking the "Edit board" button. +Once a milestone, assignee or weight is assigned to an issue board, you will no longer be able to filter through these in the search bar. In order to do that, you need to remove the desired scope -(for example, milestone, assignee, or weight) from the Issue Board. +(for example, milestone, assignee, or weight) from the issue board.  -If you don't have editing permission in a board, you're still able to see the configuration by clicking on "View scope". +If you don't have editing permission in a board, you're still able to see the configuration by +clicking **View scope**.  +--- + ### Focus mode -> - Introduced in [GitLab Starter 9.1](https://about.gitlab.com/releases/2017/04/22/gitlab-9-1-released/#issue-boards-focus-mode-ees-eep). -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212331) to GitLab Core in 12.10. +> - [Introduced]((https://about.gitlab.com/releases/2017/04/22/gitlab-9-1-released/#issue-boards-focus-mode-ees-eep)) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.1. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28597) to the Free tier of GitLab.com in 12.10. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212331) to GitLab Core in 13.0. -Click the button at the top right to toggle focus mode on and off. In focus mode, the navigation UI is hidden, allowing you to focus on issues in the board. +Click the button at the top right to toggle focus mode on and off. In focus mode, the navigation UI +is hidden, allowing you to focus on issues in the board.  -### Sum of Issue Weights **(STARTER)** +--- + +### Sum of issue weights **(STARTER)** The top of each list indicates the sum of issue weights for the issues that belong to that list. This is useful when using boards for capacity allocation, especially in combination with [assignee lists](#assignee-lists-premium). - + + +--- -### Group Issue Boards **(PREMIUM)** +### Group issue boards **(PREMIUM)** -> Introduced in [GitLab Premium 10.0](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards). +> [Introduced](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. Accessible at the group navigation level, a group issue board offers the same features as a project-level board, but it can display issues from all projects in that @@ -231,102 +282,114 @@ boards. When updating milestones and labels for an issue through the sidebar upd group-level objects are available. NOTE: **Note:** -Multiple group issue boards were originally introduced in [GitLab 10.0 Premium](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards) and -one group issue board per group was made available in GitLab 10.6 Core. +Multiple group issue boards were originally [introduced](https://about.gitlab.com/releases/2017/09/22/gitlab-10-0-released/#group-issue-boards) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0, and one group issue board per group was made available in GitLab Core 10.6.  +--- + ### Assignee lists **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5784) in GitLab 11.0 Premium. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5784) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. -Like a regular list that shows all issues that have the list label, you can add -an assignee list that shows all issues assigned to the given user. +Like in a regular list that shows all issues with a chosen label, you can add +an assignee list that shows all issues assigned to a user. You can have a board with both label lists and assignee lists. To add an assignee list: 1. Click **Add list**. 1. Select the **Assignee list** tab. -1. Search and click on the user you want to add as an assignee. +1. Search and click the user you want to add as an assignee. Now that the assignee list is added, you can assign or unassign issues to that user -by [dragging issues](#dragging-issues-between-lists) to and/or from an assignee list. +by [dragging issues](#drag-issues-between-lists) to and from an assignee list. To remove an assignee list, just as with a label list, click the trash icon.  +--- + ### Milestone lists **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6469) in GitLab 11.2 Premium. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6469) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. -As of 11.2, you're also able to create lists of a milestone. As the name states, -these are lists that filter issues by the assigned milestone, giving you more -freedom and visibility on the Issue Board. To do so: +You're also able to create lists of a milestone. These are lists that filter issues by the assigned +milestone, giving you more freedom and visibility on the issue board. To add a milestone list: 1. Click **Add list**. 1. Select the **Milestone** tab. -1. Search and click on the milestone. +1. Search and click the milestone. -Similar to the assignee lists, you're now able to [drag issues](#dragging-issues-between-lists) -to and/or from a milestone list to manipulate the milestone of the dragged issues. -As on another list types, click on the trash icon to remove it. +Similar to the assignee lists, you're now able to [drag issues](#drag-issues-between-lists) +to and from a milestone list to manipulate the milestone of the dragged issues. +As in other list types, click the trash icon to remove a list.  +--- + ## Work In Progress limits **(STARTER)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11403) in GitLab 12.7 -You can set Work In Progress (WIP) limits per issues list. When a limit is set, the list's header shows the number of issues in the list and the soft limit of issues. For example, for a list with 4 issues, and a limit of 5, the header will show `4/5`. If you exceed the limit, the current number of issues is shown in red. For example, you have a list with 5 issues with a limit of 5. When you move another issue to that list, the list's header displays `6/5`, with the `6` shown in red. +You can set Work In Progress (WIP) limits per issues list. When a limit is set, the list's header +shows the number of issues in the list and the soft limit of issues. -To set a WIP limit for a list: +Examples: -1. Navigate to a Project or Group board for which you have membership and click on the Settings icon (gear) in a list's header. -1. Next to **Work In Progress Limit**, click **Edit** and enter the maximum number of issues. Press `Enter` to save. +- You have a list with four issues, and a limit of five, the header will show **4/5**. + If you exceed the limit, the current number of issues is shown in red. +- You have a list with five issues with a limit of five. When you move another issue to that list, + the list's header displays **6/5**, with the six shown in red. -### Summary of features per tier - -Different issue board features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), as shown in the following table: +To set a WIP limit for a list: -| Tier | Number of Project Issue Boards | Number of Group Issue Boards | Configurable Issue Boards | Assignee Lists | -|----------|--------------------------------|------------------------------|---------------------------|----------------| -| Core / Free | Multiple | 1 | No | No | -| Starter / Bronze | Multiple | 1 | Yes | No | -| Premium / Silver | Multiple | Multiple | Yes | Yes | -| Ultimate / Gold | Multiple | Multiple | Yes | Yes | +1. Navigate to a Project or Group board of which you're a member. +1. Click the Settings icon (**{settings}**) in a list's header. +1. Next to **Work In Progress Limit**, click **Edit**. +1. Enter the maximum number of issues. +1. Press <kbd>Enter</kbd> to save. ## Blocked issues > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34723) in GitLab 12.8. -If an issue is blocked by another issue, an icon will display next to its title to differentiate it from unblocked issues. +If an issue is blocked by another issue, an icon appears next to its title to indicate its blocked +status.  -## Actions you can take on an Issue Board +--- + +## Actions you can take on an issue board + +- [Create a new list](#create-a-new-list). +- [Delete an existing list](#delete-a-list). +- [Add issues to a list](#add-issues-to-a-list). +- [Remove an issue from a list](#remove-an-issue-from-a-list). +- [Filter issues](#filter-issues) that appear across your issue board. +- [Create workflows](#create-workflows). +- [Drag issues between lists](#drag-issues-between-lists). +- [Mulit-select issue cards](#multi-select-issue-cards). +- [Re-order issues in lists](#issue-ordering-in-a-list). +- Drag and reorder the lists. +- Change issue labels (by dragging an issue between lists). +- Close an issue (by dragging it to the **Done** list). -- [Create a new list](#creating-a-new-list). -- [Delete an existing list](#deleting-a-list). -- Drag issues between lists. -- Re-order issues in lists. -- Drag and reorder the lists themselves. -- Change issue labels on-the-fly while dragging issues between lists. -- Close an issue if you drag it to the **Done** list. -- Create a new list from a non-existing label by [creating the label on-the-fly](#creating-a-new-list) - within the Issue Board. -- [Filter issues](#filtering-issues) that appear across your Issue Board. +If you're not able to do some of the things above, make sure you have the right +[permissions](#permissions). -If you are not able to perform one or more of the things above, make sure you -have the right [permissions](#permissions). +### First time using an issue board -### First time using the Issue Board +The first time you open an issue board, you are presented with +the default lists (**Open** and **Closed**) and a welcome message that gives +you two options. You can either: -The first time you navigate to your Issue Board, you will be presented with -a default list (**Done**) and a welcoming message that gives -you two options. You can either create a predefined set of labels and create -their corresponding lists to the Issue Board or opt-out and use your own lists. +- Create a predefined set of labels (by default: **To Do** and **Doing**) and create their + corresponding lists to the issue board. +- Opt-out and use your own lists. - + If you choose to use and create the predefined lists, they will appear as empty because the labels associated to them will not exist up until that moment, @@ -334,99 +397,74 @@ which means the system has no way of populating them automatically. That's of course if the predefined labels don't already exist. If any of them does exist, the list will be created and filled with the issues that have that label. -### Creating a new list +### Create a new list -Create a new list by clicking on the **Add list** button at the upper -right corner of the Issue Board. +Create a new list by clicking the **Add list** button in the upper right corner of the issue board. - + -Simply choose the label or user to create the list from. The new list will be inserted +Then, choose the label or user to create the list from. The new list will be inserted at the end of the lists, before **Done**. Moving and reordering lists is as easy as dragging them around. -To create a list for a label that doesn't yet exist, simply create the label by -choosing **Create new label**. The label will be created on-the-fly and it will -be immediately added to the dropdown. You can now choose it to create a list. +To create a list for a label that doesn't yet exist, create the label by +choosing **Create new label**. This creates the label immediately and adds it to the dropdown. +You can now choose it to create a list. -### Deleting a list +### Delete a list -To delete a list from the Issue Board use the small trash icon that is present +To delete a list from the issue board, use the small trash icon present in the list's heading. A confirmation dialog will appear for you to confirm. Deleting a list doesn't have any effect in issues and labels, it's just the list view that is removed. You can always add it back later if you need. -### Adding issues to a list +### Add issues to a list -You can add issues to a list by clicking the **Add issues** button that is -present in the upper right corner of the Issue Board. This will open up a modal +You can add issues to a list by clicking the **Add issues** button +present in the upper right corner of the issue board. This will open up a modal window where you can see all the issues that do not belong to any list. -Select one or more issues by clicking on the cards and then click **Add issues** +Select one or more issues by clicking the cards and then click **Add issues** to add them to the selected list. You can limit the issues you want to add to the list by filtering by author, assignee, milestone, and label.  -### Removing an issue from a list - -Removing an issue from a list can be done by clicking on the issue card and then -clicking the **Remove from board** button in the sidebar. Under the hood, the -respective label is removed, and as such it's also removed from the list and the -board itself. - - - -### Issue ordering in a list +--- -When visiting a board, issues appear ordered in any list. You are able to change -that order simply by dragging and dropping the issues. The changed order will be saved -to the system so that anybody who visits the same board later will see the reordering, -with some exceptions. +### Remove an issue from a list -The first time a given issue appears in any board (that is, the first time a user -loads a board containing that issue), it is ordered with -respect to other issues in that list according to [Priority order](labels.md#label-priority). +Removing an issue from a list can be done by clicking the issue card and then +clicking the **Remove from board** button in the sidebar. The +respective label is removed. -At that point, that issue is assigned a relative order value by the system -representing its relative order with respect to the other issues in the list. Any time -you drag-and-drop reorder that issue, its relative order value changes accordingly. - -Also, any time that issue appears in any board when it's loaded by a user, -the updated relative order value is used for the ordering. (It's only the first -time an issue appears that it takes from the Priority order mentioned above.) This means that -if issue `A` is drag-and-drop reordered to be above issue `B` by any user in -a given board inside your GitLab instance, any time those two issues are subsequently -loaded in any board in the same instance (could be a different project board or a different group -board, for example), that ordering is maintained. + -This ordering also affects [issue lists](issues/sorting_issue_lists.md). -Changing the order in an issue board changes the ordering in an issue list, -and vice versa. +--- -### Filtering issues +### Filter issues -You should be able to use the filters on top of your Issue Board to show only +You should be able to use the filters on top of your issue board to show only the results you want. This is similar to the filtering used in the issue tracker -since the metadata from the issues and labels are re-used in the Issue Board. +since the metadata from the issues and labels are re-used in the issue board. You can filter by author, assignee, milestone, and label. -### Creating workflows +### Create workflows -By reordering your lists, you can create workflows. As lists in Issue Boards are +By reordering your lists, you can create workflows. As lists in issue boards are based on labels, it works out of the box with your existing issues. So if you've already labeled things with 'Backend' and 'Frontend', the issue appears in the lists as you create them. In addition, this means you can easily move something between lists by changing a label. -A typical workflow of using the Issue Board would be: +A typical workflow of using an issue board would be: 1. You have [created](labels.md#label-management) and [prioritized](labels.md#label-priority) labels so that you can easily categorize your issues. 1. You have a bunch of issues (ideally labeled). -1. You visit the Issue Board and start [creating lists](#creating-a-new-list) to +1. You visit the issue board and start [creating lists](#create-a-new-list) to create a workflow. 1. You move issues around in lists so that your team knows who should be working on what issue. @@ -446,9 +484,11 @@ issue. This process can be seen clearly when visiting an issue since with every move to another list the label changes and a system not is recorded. - + -### Dragging issues between lists +--- + +### Drag issues between lists When dragging issues between lists, different behavior occurs depending on the source list and the target list. @@ -472,6 +512,35 @@ To select and move multiple cards:  +--- + +### Issue ordering in a list + +When visiting a board, issues appear ordered in any list. You're able to change +that order by dragging and dropping the issues. The changed order will be saved +to the system so that anybody who visits the same board later will see the reordering, +with some exceptions. + +The first time a given issue appears in any board (that is, the first time a user +loads a board containing that issue), it is ordered with +respect to other issues in that list according to [Priority order](labels.md#label-priority). + +At that point, that issue is assigned a relative order value by the system +representing its relative order with respect to the other issues in the list. Any time +you drag-and-drop reorder that issue, its relative order value changes accordingly. + +Also, any time that issue appears in any board when it's loaded by a user, +the updated relative order value is used for the ordering. (It's only the first +time an issue appears that it takes from the Priority order mentioned above.) This means that +if issue `A` is drag-and-drop reordered to be above issue `B` by any user in +a given board inside your GitLab instance, any time those two issues are subsequently +loaded in any board in the same instance (could be a different project board or a different group +board, for example), that ordering is maintained. + +This ordering also affects [issue lists](issues/sorting_issue_lists.md). +Changing the order in an issue board changes the ordering in an issue list, +and vice versa. + ## Tips A few things to remember: @@ -480,8 +549,8 @@ A few things to remember: and adds the label from the list it goes to. - An issue can exist in multiple lists if it has more than one label. - Lists are populated with issues automatically if the issues are labeled. -- Clicking on the issue title inside a card takes you to that issue. -- Clicking on a label inside a card quickly filters the entire Issue Board +- Clicking the issue title inside a card takes you to that issue. +- Clicking a label inside a card quickly filters the entire issue board and show only the issues from all lists that have that label. - For performance and visibility reasons, each list shows the first 20 issues by default. If you have more than 20 issues, start scrolling down and the next diff --git a/doc/user/project/issues/csv_export.md b/doc/user/project/issues/csv_export.md index ec53b3dbbba..af01534a67f 100644 --- a/doc/user/project/issues/csv_export.md +++ b/doc/user/project/issues/csv_export.md @@ -70,7 +70,7 @@ Data will be encoded with a comma as the column delimiter, with `"` used to quot | Labels | Title of any labels joined with a `,` | | Time Estimate | [Time estimate](../time_tracking.md#estimates) in seconds | | Time Spent | [Time spent](../time_tracking.md#time-spent) in seconds | -| Epic ID | Id of the parent epic **(ULTIMATE)**, introduced in 12.7 | +| Epic ID | ID of the parent epic **(ULTIMATE)**, introduced in 12.7 | | Epic Title | Title of the parent epic **(ULTIMATE)**, introduced in 12.7 | ## Limitations diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index 1078d0410ed..64f56221632 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -1,6 +1,7 @@ -# Design Management **(PREMIUM)** +# Design Management -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/660) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/660) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212566) to GitLab Core in 13.0. CAUTION: **Warning:** This an **alpha** feature and is subject to change at any time without @@ -86,6 +87,7 @@ Copy-and-pasting has some limitations: - You can paste only one image at a time. When copy/pasting multiple files, only the first one will be uploaded. - All images will be converted to `png` format under the hood, so when you want to copy/paste `gif` file, it will result in broken animation. +- If you are pasting a screenshot from the clipboard, it will be renamed to `design_<timestamp>.png` - Copy/pasting designs is not supported on Internet Explorer. Designs with the same filename as an existing uploaded design will create a new version diff --git a/doc/user/project/issues/due_dates.md b/doc/user/project/issues/due_dates.md index f70597f6875..0be0cdd11bd 100644 --- a/doc/user/project/issues/due_dates.md +++ b/doc/user/project/issues/due_dates.md @@ -24,6 +24,11 @@ Changes are saved immediately.  +The last way to set a due date is by using [quick actions](../quick_actions.md), directly in an issue's description or comment: + +- `/due <date>`: set due date. Examples of valid `<date>` include `in 2 days`, `this Friday`, and `December 31st`. +- `/remove_due_date`: remove due date. + ## Making use of due dates Issues that have a due date can be easily seen in the issue tracker, diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 0f9295e1afd..22221531360 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -28,6 +28,11 @@ you can also view all the issues collectively at the group level. See also [Always start a discussion with an issue](https://about.gitlab.com/blog/2016/03/03/start-with-an-issue/). +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +To learn how GitLab's Strategic Marketing department uses GitLab issues with [labels](../labels.md) and +[issue boards](../issue_board.md), see the video on +[Managing Commitments with Issues](https://www.youtube.com/watch?v=cuIHNintg1o&t=3). + ## Parts of an issue Issues contain a variety of content and metadata, enabling a large range of flexibility diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 4ee29f3357d..4e329889e7c 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -50,7 +50,7 @@ create issues for the same project.  -### New issue via Service Desk **(PREMIUM)** +### New issue via Service Desk **(STARTER)** Enable [Service Desk](../service_desk.md) for your project and offer email support. By doing so, when your customer sends a new email, a new issue can be created in @@ -92,9 +92,17 @@ field values using query string parameters in a URL. This is useful for embeddin a URL in an external HTML page, and also certain scenarios where you want the user to create an issue with certain fields prefilled. -The title, description, and description template fields can be prefilled using -this method. You cannot pre-fill both the description and description template fields -in the same URL (since a description template also populates the description field). +The title, description, description template, and confidential fields can be prefilled +using this method. You cannot pre-fill both the description and description template +fields in the same URL (since a description template also populates the description +field). + +| Field | URL Parameter Name | Notes | +|----------------------|-----------------------|-------------------------------------------------------| +| title | `issue[title]` | | +| description | `issue[description]` | | +| description template | `issuable_template` | | +| confidential | `issue[confidential]` | Parameter value must be `true` to set to confidential | Follow these examples to form your new issue URL with prefilled fields. @@ -102,6 +110,8 @@ Follow these examples to form your new issue URL with prefilled fields. and a pre-filled description, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea` - For a new issue in the GitLab Community Edition project with a pre-filled title and a pre-filled description template, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/issues/new?issue[title]=Validate%20new%20concept&issuable_template=Research%20proposal` +- For a new issue in the GitLab Community Edition project with a pre-filled title, + a pre-filled description, and the confidential flag set, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea&issue[confidential]=true` ## Moving Issues @@ -117,7 +127,10 @@ The "Move issue" button is at the bottom of the right-sidebar when viewing the i If you have advanced technical skills you can also bulk move all the issues from one project to another in the rails console. The below script will move all the issues from one project to another that are not in status **closed**. -To access rails console run `sudo gitlab-rails console` on the GitLab server and run the below script. Please be sure to change **project**, **admin_user** and **target_project** to your values. We do also recommend [creating a backup](../../../raketasks/backup_restore.md#creating-a-backup-of-the-gitlab-system) before attempting any changes in the console. +To access rails console run `sudo gitlab-rails console` on the GitLab server and run the below +script. Please be sure to change **project**, **admin_user** and **target_project** to your values. +We do also recommend [creating a backup](../../../raketasks/backup_restore.md#back-up-gitlab) before +attempting any changes in the console. ```ruby project = Project.find_by_full_path('full path of the project where issues are moved from') @@ -170,7 +183,7 @@ but it will not close automatically. If the issue is in a different repository than the MR, add the full URL for the issue(s): -```md +```markdown Closes #4, #6, and https://gitlab.com/<username>/<projectname>/issues/<xxx> ``` @@ -179,7 +192,7 @@ Closes #4, #6, and https://gitlab.com/<username>/<projectname>/issues/<xxx> When not specified, the default issue closing pattern as shown below will be used: ```shell -((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)|[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+) +\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+) ``` This translates to the following keywords: diff --git a/doc/user/project/issues/related_issues.md b/doc/user/project/issues/related_issues.md index 5fba73c2971..8002b528a80 100644 --- a/doc/user/project/issues/related_issues.md +++ b/doc/user/project/issues/related_issues.md @@ -10,12 +10,14 @@ The relationship only shows up in the UI if the user can see both issues. ## Adding a related issue +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2035) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/34239) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.0. +> When you try to close an issue with open blockers, you'll see a warning that you can dismiss. + You can relate one issue to another by clicking the related issues "+" button in the header of the related issue block. Then, input the issue reference number or paste in the full URL of the issue. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2035) in GitLab 12.8. - Additionally, you can select whether the current issue relates to, blocks, or is blocked by the issues being entered.  diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index 9cf50d3aaed..f3b59147d5b 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -2,13 +2,18 @@ ## Overview -Labels allow you to categorize epics, issues, and merge requests using descriptive titles like -`bug`, `feature request`, or `docs`, as well as customizable colors. They allow you to quickly -and dynamically filter and manage epics, issues, and merge requests, and are a key -part of [issue boards](issue_board.md). +As your count of issues, merge requests, and epics grows in GitLab, it's more and more challenging +to keep track of those items. Especially as your organization grows from just a few people to +hundreds or thousands. This is where labels come in. They help you organize and tag your work +so you can track and find the work items you're interested in. -You can use labels to help [search](../search/index.md#issues-and-merge-requests) in -lists of issues, merge requests, and epics, as well as [search in issue boards](../search/index.md#issue-boards). +Labels are a key part of [issue boards](issue_board.md). With labels you can: + +- Categorize epics, issues, and merge requests using colors and descriptive titles like +`bug`, `feature request`, or `docs`. +- Dynamically filter and manage epics, issues, and merge requests. +- [Search lists of issues, merge requests, and epics](../search/index.md#issues-and-merge-requests), + as well as [issue boards](../search/index.md#issue-boards). ## Project labels and group labels @@ -164,7 +169,7 @@ Suppose you have the labels `workflow::development`, `workflow::review`, and applied, and a developer wanted to advance the issue to `workflow::review`, they would simply apply that label, and the `workflow::development` label would automatically be removed. This behavior already exists when you move issues -across label lists in an [issue board](issue_board.md#creating-workflows), but +across label lists in an [issue board](issue_board.md#create-workflows), but now, team members who may not be working in an issue board directly would still be able to advance workflow states consistently in issues themselves. diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index 27a5701e6c2..9020dc335b6 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -1,4 +1,4 @@ -# Project's members +# Members of a project You can manage the groups and users and their access levels in all of your projects. You can also personalize the access level you give each user, diff --git a/doc/user/project/merge_requests/accessibility_testing.md b/doc/user/project/merge_requests/accessibility_testing.md index 755bf0447e3..427761281f6 100644 --- a/doc/user/project/merge_requests/accessibility_testing.md +++ b/doc/user/project/merge_requests/accessibility_testing.md @@ -18,6 +18,23 @@ measuring the accessibility of web sites, and has built a simple This job outputs accessibility violations, warnings, and notices for each page analyzed to a file called `accessibility`. +## Accessibility Merge Request widget + +[Since GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/39425), in addition to the report artifact that is created, GitLab will also show the +Accessibility Report in the merge request widget area: + + + +This widget comes with the `:accessibility_report_view` feature flag disabled by default while we test feature stability. +Once we have determined the widget is stable, this feature will be enabled by default. + +To enable this feature, ask a GitLab administrator with [Rails console access](../../../administration/feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the +following command: + +```ruby +Feature.enable(:accessibility_report_view) +``` + ## Configure Accessibility Testing This example shows how to run [pa11y](https://pa11y.org/) @@ -46,10 +63,13 @@ Pa11y against the webpages defined in `a11y_urls`, and builds an HTML report for The report for each URL is saved as an artifact that can be [viewed directly in your browser](../../../ci/pipelines/job_artifacts.md#browsing-artifacts). -A single `accessibility.json` artifact is created and saved along with the individual HTML reports. +A single `gl-accessibility.json` artifact is created and saved along with the individual HTML reports. It includes report data for all URLs scanned. NOTE: **Note:** +For GitLab 12.10 and earlier, the [artifact generated is named `accessibility.json`](https://gitlab.com/gitlab-org/ci-cd/accessibility/-/merge_requests/9). + +NOTE: **Note:** For GitLab versions earlier than 12.9, you can use `include:remote` and use a link to the [current template in `master`](https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml) diff --git a/doc/user/project/merge_requests/browser_performance_testing.md b/doc/user/project/merge_requests/browser_performance_testing.md index 1bca5d2a212..0fa3d7be265 100644 --- a/doc/user/project/merge_requests/browser_performance_testing.md +++ b/doc/user/project/merge_requests/browser_performance_testing.md @@ -6,55 +6,50 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3507) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.3. -If your application offers a web interface and you are using +If your application offers a web interface and you're using [GitLab CI/CD](../../../ci/README.md), you can quickly determine the performance impact of pending code changes. ## Overview GitLab uses [Sitespeed.io](https://www.sitespeed.io), a free and open source -tool for measuring the performance of web sites, and has built a simple -[Sitespeed plugin](https://gitlab.com/gitlab-org/gl-performance) -which outputs the results in a file called `performance.json`. This plugin -outputs the performance score for each page that is analyzed. - +tool, for measuring the performance of web sites. GitLab has built a simple +[Sitespeed plugin](https://gitlab.com/gitlab-org/gl-performance) which outputs +the performance score for each page analyzed in a file called `performance.json`. The [Sitespeed.io performance score](https://examples.sitespeed.io/6.0/2017-11-23-23-43-35/help.html) -is a composite value based on best practices, and we will be expanding support -for [additional metrics](https://gitlab.com/gitlab-org/gitlab/issues/4370) -in a future release. +is a composite value based on best practices. -Going a step further, GitLab can show the Performance report right -in the merge request widget area (see below). +GitLab can [show the Performance report](#how-browser-performance-testing-works) +in the merge request widget area. ## Use cases -For instance, consider the following workflow: +Consider the following workflow: 1. A member of the marketing team is attempting to track engagement by adding a new tool. 1. With browser performance metrics, they see how their changes are impacting the usability of the page for end users. -1. The metrics show that after their changes the performance score of the page has gone down. -1. When looking at the detailed report, they see that the new JavaScript library was - included in `<head>` which affects loading page speed. -1. They ask a front end developer to help them, who sets the library to load asynchronously. -1. The frontend developer approves the merge request and authorizes its deployment to production. - -## How it works +1. The metrics show that after their changes, the performance score of the page has gone down. +1. When looking at the detailed report, they see the new JavaScript library was + included in `<head>`, which affects loading page speed. +1. They ask for help from a front end developer, who sets the library to load asynchronously. +1. The frontend developer approves the merge request, and authorizes its deployment to production. -First of all, you need to define a job in your `.gitlab-ci.yml` file that generates the -[Performance report artifact](../../../ci/yaml/README.md#artifactsreportsperformance-premium). -For more information on how the Performance job should look like, check the -example on [Configuring Browser Performance Testing](#configuring-browser-performance-testing). +## How browser performance testing works +First, define a job in your `.gitlab-ci.yml` file that generates the +[Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsperformance-premium). GitLab then checks this report, compares key performance metrics for each page -between the source and target branches, and shows the information right on the merge request. +between the source and target branches, and shows the information in the merge request. + +For an example Performance job, see +[Configuring Browser Performance Testing](#configuring-browser-performance-testing). NOTE: **Note:** -If the Performance report doesn't have anything to compare to, no information -will be displayed in the merge request area. That is the case when you add the -Performance job in your `.gitlab-ci.yml` for the very first time. -Consecutive merge requests will have something to compare to, and the Performance -report will be shown properly. +If the Performance report has no data to compare, such as when you add the +Performance job in your `.gitlab-ci.yml` for the very first time, no information +displays in the merge request widget area. Consecutive merge requests will have data for +comparison, and the Performance report will be shown properly.  @@ -64,52 +59,51 @@ This example shows how to run the [sitespeed.io container](https://hub.docker.co on your code by using GitLab CI/CD and [sitespeed.io](https://www.sitespeed.io) using Docker-in-Docker. -First, you need GitLab Runner with -[docker-in-docker build](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). +1. First, set up GitLab Runner with a + [docker-in-docker build](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). +1. After configuring the Runner, add a new job to `.gitlab-ci.yml` that generates + the expected report. +1. Define the `performance` job according to your version of GitLab: -Once you set up the Runner, add a new job to `.gitlab-ci.yml` that generates the -expected report. + - For GitLab 12.4 and later - [include](../../../ci/yaml/README.md#includetemplate) the + [`Browser-Performance.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml) provided as a part of your GitLab installation. + - For GitLab versions earlier than 12.4 - Copy and use the job as defined in the + [`Browser-Performance.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml). -For GitLab 12.4 and later, to define the `performance` job, you must -[include](../../../ci/yaml/README.md#includetemplate) the -[`Browser-Performance.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml) -that's provided as a part of your GitLab installation. -For GitLab versions earlier than 12.4, you can copy and use the job as defined -in that template. + CAUTION: **Caution:** + The job definition provided by the template does not support Kubernetes yet. + For a complete example of a more complex setup that works in Kubernetes, see + [`Browser-Performance-Testing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml). -CAUTION: **Caution:** -The job definition provided by the template does not support Kubernetes yet. For a complete example of a more complex setup -that works in Kubernetes, see [here](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml). +1. Add the following to your `.gitlab-ci.yml` file: -Add the following to your `.gitlab-ci.yml` file: + ```yaml + include: + template: Verify/Browser-Performance.gitlab-ci.yml -```yaml -include: - template: Verify/Browser-Performance.gitlab-ci.yml + performance: + variables: + URL: https://example.com + ``` -performance: - variables: - URL: https://example.com -``` - -CAUTION: **Caution:** -The job definition provided by the template is supported in GitLab 11.5 and later versions. -It also requires GitLab Runner 11.5 or later. For earlier versions, use the -[previous job definitions](#previous-job-definitions). + CAUTION: **Caution:** + The job definition provided by the template is supported in GitLab 11.5 and later versions. + It also requires GitLab Runner 11.5 or later. For earlier versions, use the + [previous job definitions](#previous-job-definitions). -The above example will create a `performance` job in your CI/CD pipeline and will run +The above example creates a `performance` job in your CI/CD pipeline and runs sitespeed.io against the webpage you defined in `URL` to gather key metrics. The [GitLab plugin for sitespeed.io](https://gitlab.com/gitlab-org/gl-performance) -is downloaded in order to save the report as a [Performance report artifact](../../../ci/yaml/README.md#artifactsreportsperformance-premium) -that you can later download and analyze. Due to implementation limitations we always +is downloaded to save the report as a [Performance report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportsperformance-premium) +that you can later download and analyze. Due to implementation limitations, we always take the latest Performance artifact available. -The full HTML sitespeed.io report will also be saved as an artifact, and if you have -[GitLab Pages](../pages/index.md) enabled, it can be viewed directly in your browser. +The full HTML sitespeed.io report is saved as an artifact, and if +[GitLab Pages](../pages/index.md) is enabled, it can be viewed directly in your browser. -It is also possible to customize options by setting the `SITESPEED_OPTIONS` variable. -For example, this is how to override the number of runs sitespeed.io -will make on the given URL: +You can also customize options by setting the `SITESPEED_OPTIONS` variable. +For example, you can override the number of runs sitespeed.io +makes on the given URL: ```yaml include: @@ -122,27 +116,47 @@ performance: ``` For further customization options for sitespeed.io, including the ability to provide a -list of URLs to test, please see the [Sitespeed.io Configuration](https://www.sitespeed.io/documentation/sitespeed.io/configuration/) +list of URLs to test, please see the +[Sitespeed.io Configuration](https://www.sitespeed.io/documentation/sitespeed.io/configuration/) documentation. TIP: **Tip:** Key metrics are automatically extracted and shown in the merge request widget. +### Configuring degradation threshold + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27599) in GitLab 13.0. + +You can configure the sensitivity of degradation alerts to avoid getting alerts for minor drops in metrics. +This is done by setting the `DEGRADATION_THRESHOLD` variable. In the example below, the alert will only show up +if the `Total Score` metric degrades by 5 points or more: + +```yaml +include: + template: Verify/Browser-Performance.gitlab-ci.yml + +performance: + variables: + URL: https://example.com + DEGRADATION_THRESHOLD: 5 +``` + +The `Total Score` metric is based on sitespeed.io's [coach performance score](https://www.sitespeed.io/documentation/sitespeed.io/metrics/#performance-score). There is more information in [the coach documentation](https://www.sitespeed.io/documentation/coach/how-to/#what-do-the-coach-do). + ### Performance testing on Review Apps -The above CI YML is great for testing against static environments, and it can -be extended for dynamic environments. There are a few extra steps to take to -set this up: +The above CI YAML configuration is great for testing against static environments, and it can +be extended for dynamic environments, but a few extra steps are required: 1. The `performance` job should run after the dynamic environment has started. 1. In the `review` job, persist the hostname and upload it as an artifact so - it's available to the `performance` job (the same can be done for static - environments like staging and production to unify the code path). Saving it - as an artifact is as simple as `echo $CI_ENVIRONMENT_URL > environment_url.txt` + it's available to the `performance` job. The same can be done for static + environments like staging and production to unify the code path. You can save it + as an artifact with `echo $CI_ENVIRONMENT_URL > environment_url.txt` in your job's `script`. 1. In the `performance` job, read the previous artifact into an environment - variable, in this case `$URL` because this is what our sitespeed.io command - uses for the URL parameter. Because Review App URLs are dynamic, we define + variable. In this case, use `$URL` because the sitespeed.io command + uses it for the URL parameter. Because Review App URLs are dynamic, define the `URL` variable through `before_script` instead of `variables`. 1. You can now run the sitespeed.io container against the desired hostname and paths. @@ -183,11 +197,11 @@ performance: ### Previous job definitions CAUTION: **Caution:** -Before GitLab 11.5, Performance job and artifact had to be named specifically +Before GitLab 11.5, the Performance job and artifact had to be named specifically to automatically extract report data and show it in the merge request widget. -While these old job definitions are still maintained they have been deprecated +While these old job definitions are still maintained, they have been deprecated and may be removed in next major release, GitLab 12.0. -You are advised to update your current `.gitlab-ci.yml` configuration to reflect that change. +GitLab recommends you update your current `.gitlab-ci.yml` configuration to reflect that change. For GitLab 11.4 and earlier, the job should look like: diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index a7e712a4c0a..beb90e30906 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -27,7 +27,19 @@ in the merge request widget area:  -For more information, see the Code Climate list of [Supported Languages for Maintainability](https://docs.codeclimate.com/docs/supported-languages-for-maintainability). +Watch a quick walkthrough of Code Quality in action: + +<div class="video-fallback"> + See the video: <a href="https://www.youtube.com/watch?v=B32LxtJKo9M">Code Quality: Speed Run</a>. +</div> +<figure class="video-container"> + <iframe src="https://www.youtube.com/embed/B32LxtJKo9M" frameborder="0" allowfullscreen="true"> </iframe> +</figure> + +NOTE: **Note:** +For one customer, the auditor found that having Code Quality, SAST, and Container Scanning all automated in GitLab CI/CD was almost better than a manual review! [Read more](https://about.gitlab.com/customers/bi_worldwide/). + +See also the Code Climate list of [Supported Languages for Maintainability](https://docs.codeclimate.com/docs/supported-languages-for-maintainability). ## Use cases @@ -37,7 +49,7 @@ For instance, consider the following workflow: feature in your app faster. 1. With Code Quality reports, they analyze how their implementation is impacting the code quality. -1. The metrics show that their code degrade the quality in 10 points. +1. The metrics show that their code degrades the quality by 10 points. 1. You ask a co-worker to help them with this modification. 1. They both work on the changes until Code Quality report displays no degradations, only improvements. @@ -53,10 +65,10 @@ also requires the GitLab Runner 11.5 or later. For earlier versions, use the This example shows how to run Code Quality on your code by using GitLab CI/CD and Docker. -First, you need GitLab Runner with: +First, you need GitLab Runner configured: -- The [docker-in-docker executor](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). -- Enough disk space to handle generated Code Quality files. For example on the [GitLab project](https://gitlab.com/gitlab-org/gitlab) the files are approximately 7 GB. +- For the [docker-in-docker workflow](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). +- With enough disk space to handle generated Code Quality files. For example on the [GitLab project](https://gitlab.com/gitlab-org/gitlab) the files are approximately 7 GB. Once you set up the Runner, include the CodeQuality template in your CI config: @@ -67,7 +79,7 @@ include: The above example will create a `code_quality` job in your CI/CD pipeline which will scan your source code for code quality issues. The report will be saved as a -[Code Quality report artifact](../../../ci/yaml/README.md#artifactsreportscodequality-starter) +[Code Quality report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality-starter) that you can later download and analyze. Due to implementation limitations we always take the latest Code Quality artifact available. @@ -227,7 +239,7 @@ do this: 1. Define a job in your `.gitlab-ci.yml` file that generates the [Code Quality report - artifact](../../../ci/yaml/README.md#artifactsreportscodequality-starter). + artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality-starter). 1. Configure your tool to generate the Code Quality report artifact as a JSON file that implements subset of the [Code Climate spec](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types). diff --git a/doc/user/project/merge_requests/creating_merge_requests.md b/doc/user/project/merge_requests/creating_merge_requests.md index d8a2b427288..544727380e7 100644 --- a/doc/user/project/merge_requests/creating_merge_requests.md +++ b/doc/user/project/merge_requests/creating_merge_requests.md @@ -171,7 +171,7 @@ When the changes are merged, your changes are added to the upstream repository a the branch as per specification. After your work is merged, if you don't intend to make any other contributions to the upstream project, you can unlink your fork from its upstream project in the **Settings > Advanced Settings** section by -[removing the forking relashionship](../settings/index.md#removing-a-fork-relationship). +[removing the forking relationship](../settings/index.md#removing-a-fork-relationship). For further details, [see the forking workflow documentation](../repository/forking_workflow.md). diff --git a/doc/user/project/merge_requests/img/accessibility_mr_widget_v13_0.png b/doc/user/project/merge_requests/img/accessibility_mr_widget_v13_0.png Binary files differnew file mode 100644 index 00000000000..c52bf9964f1 --- /dev/null +++ b/doc/user/project/merge_requests/img/accessibility_mr_widget_v13_0.png diff --git a/doc/user/project/merge_requests/img/code_quality.png b/doc/user/project/merge_requests/img/code_quality.png Binary files differindex a20f6476fb8..3c6c92baad2 100644 --- a/doc/user/project/merge_requests/img/code_quality.png +++ b/doc/user/project/merge_requests/img/code_quality.png diff --git a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md index bb5aadfa9b9..fdcb1049ef7 100644 --- a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md +++ b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md @@ -64,6 +64,15 @@ list.  +### Merge requests commit navigation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18140) in GitLab 13.0. + +To seamlessly navigate among commits in a merge request, from the **Commits** tab, click one of +the commits to open the single-commit view. From there, you can navigate among the commits +by clicking the **Prev** and **Next** buttons on the top-right of the page or by using the +<kbd>X</kbd> and <kbd>C</kbd> keyboard shortcuts. + ### Incrementally expand merge request diffs By default, the diff shows only the parts of a file which are changed. @@ -102,7 +111,7 @@ you will be able to see: - Both pre and post-merge pipelines and the environment information if any. - Which deployments are in progress. -If there's an [environment](../../../ci/environments.md) and the application is +If there's an [environment](../../../ci/environments/index.md) and the application is successfully deployed to it, the deployed environment and the link to the Review App will be shown as well. diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index 71fbdaf112f..84d60fca72d 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -17,14 +17,14 @@ MR is merged. ## How test coverage visualization works Collecting the coverage information is done via GitLab CI/CD's -[artifacts reports feature](../../../ci/yaml/README.md#artifactsreports). +[artifacts reports feature](../../../ci/pipelines/job_artifacts.md#artifactsreports). You can specify one or more coverage reports to collect, including wildcard paths. GitLab will then take the coverage information in all the files and combine it together. For the coverage analysis to work, you have to provide a properly formatted [Cobertura XML](https://cobertura.github.io/cobertura/) report to -[`artifacts:reports:cobertura`](../../../ci/yaml/README.md#artifactsreportscobertura). +[`artifacts:reports:cobertura`](../../../ci/pipelines/job_artifacts.md#artifactsreportscobertura). This format was originally developed for Java, but most coverage analysis frameworks for other languages have plugins to add support for it, like: diff --git a/doc/user/project/merge_requests/versions.md b/doc/user/project/merge_requests/versions.md index 2f51af24a95..84934148bdc 100644 --- a/doc/user/project/merge_requests/versions.md +++ b/doc/user/project/merge_requests/versions.md @@ -57,7 +57,7 @@ source and target branch can be shown mixed together making it hard to understand which changes are being added and which already exist in the target branch. -In GitLab 12.10, we added an **experimental** comparison mode, which +In GitLab 12.10, we added a comparison mode, which shows a diff calculated by simulating how it would look like once merged - a more accurate representation of the changes rather than using the base of the two branches. The new mode is available from the comparison target drop down @@ -67,26 +67,6 @@ current default comparison.  -### Enable or disable `HEAD` comparison mode **(CORE ONLY)** - -`HEAD` comparison mode is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/troubleshooting/navigating_gitlab_via_rails_console.md#starting-a-rails-console-session) -can enable it for your instance. You're welcome to test it, but use it at your -own risk. - -To enable it: - -```ruby -Feature.enable(:diff_compare_with_head) -``` - -To disable it: - -```ruby -Feature.disable(:diff_compare_with_head) -``` - <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/new_ci_build_permissions_model.md b/doc/user/project/new_ci_build_permissions_model.md index c5ab8d66852..120c7a35cb2 100644 --- a/doc/user/project/new_ci_build_permissions_model.md +++ b/doc/user/project/new_ci_build_permissions_model.md @@ -229,5 +229,5 @@ to projects and their project permissions. ### API -GitLab API cannot be used via `CI_JOB_TOKEN` but there is a [proposal](https://gitlab.com/gitlab-org/gitlab-foss/issues/29566) +GitLab API cannot be used via `CI_JOB_TOKEN` but there is a [proposal](https://gitlab.com/gitlab-org/gitlab/-/issues/35067) to support it. diff --git a/doc/user/project/operations/alert_management.md b/doc/user/project/operations/alert_management.md new file mode 100644 index 00000000000..2dcf72aaf01 --- /dev/null +++ b/doc/user/project/operations/alert_management.md @@ -0,0 +1,79 @@ +# Alert Management + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2877) in GitLab 13.0. + +Alert Management enables developers to easily discover and view the alerts +generated by their application. By surfacing alert information where the code is +being developed, efficiency and awareness can be increased. + +## Enable Alert Management + +NOTE: **Note:** +You will need at least Maintainer [permissions](../../permissions.md) to enable the Alert Management feature. + +1. Follow the [instructions for toggling generic alerts](../integrations/generic_alerts.md#setting-up-generic-alerts) +1. You can now visit **{cloud-gear}** **Operations > Alerts** in your project's sidebar to [view a list](#alert-management-list) of alerts. + + + +## Populate Alert data + +To populate data, see the instructions for +[customizing the payload](../integrations/generic_alerts.md) and making a +request to the alerts endpoint. + +## Alert Management severity + +Each level of alert contains a uniquely shaped and color-coded icon to help +you identify the severity of a particular alert. These severity icons help you +immediately identify which alerts you should prioritize investigating: + + + +Alerts contain one of the following icons: + +- **Critical**: **{severity-critical}** and hexadecimal color `#8b2615` +- **High**: **{severity-high}** and hexadecimal color `#c0341d` +- **Medium**: **{severity-medium}** and hexadecimal color `#fca429` +- **Low**: **{severity-low}** and hexadecimal color `#fdbc60` +- **Info**: **{severity-info}** and hexadecimal color `#418cd8` +- **Unknown**: **{severity-unknown}** and hexadecimal color `#bababa` + +## Alert Management list + +NOTE: **Note:** +You will need at least Developer [permissions](../../permissions.md) to view the Alert Management list. + +You can find the Alert Management list at **{cloud-gear}** **Operations > Alerts** in your project's sidebar. +Each alert contains the following metrics: + + + +- **Severity** - The current importance of a alert and how much attention it should receive. +- **Start time** - How long ago the alert fired. This field uses the standard GitLab pattern of `X time ago`, but is supported by a granular date/time tooltip depending on the user's locale. +- **End time** - How long ago the alert fired was resolved. This field uses the standard GitLab pattern of `X time ago`, but is supported by a granular date/time tooltip depending on the user's locale. +- **Alert description** - The description of the alert, which attempts to capture the most meaningful data. +- **Event count** - The number of times that an alert has fired. +- **Status** - The [current status](#alert-management-statuses) of the alert. + +### Alert Management statuses + +Each alert contains a status dropdown to indicate which alerts need investigation. +Standard alert statuses include `triggered`, `acknowledged`, and `resolved`: + +- **Triggered**: No one has begun investigation. +- **Acknowledged**: Someone is actively investigating the problem. +- **Resolved**: No further work is required. + +## Alert Management details + +NOTE: **Note:** +You will need at least Developer [permissions](../../permissions.md) to view Alert Management details. + +Navigate to the Alert Management detail view by visiting the [Alert Management list](#alert-management-list) and selecting an Alert from the list. + + + +### Update an Alert's status + +The Alert Management detail view allows users to update the Alert Status. See [Alert Management statuses](#alert-management-statuses) for more details. diff --git a/doc/user/project/operations/error_tracking.md b/doc/user/project/operations/error_tracking.md index a95459e093a..23a50fd7766 100644 --- a/doc/user/project/operations/error_tracking.md +++ b/doc/user/project/operations/error_tracking.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Error Tracking > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/169) in GitLab 11.8. @@ -53,7 +59,7 @@ From error list, users can navigate to the error details page by clicking the ti This page has: - A link to the Sentry issue. -- A link to the GitLab commit if the Sentry [release id/version](https://docs.sentry.io/workflow/releases/?platform=javascript#configure-sdk) on the Sentry Issue's first release matches a commit SHA in your GitLab hosted project. +- A link to the GitLab commit if the Sentry [release ID/version](https://docs.sentry.io/workflow/releases/?platform=javascript#configure-sdk) on the Sentry Issue's first release matches a commit SHA in your GitLab hosted project. - Other details about the issue, including a full stack trace. - In [GitLab 12.7 and newer](https://gitlab.com/gitlab-org/gitlab/issues/36246), language and urgency are displayed. diff --git a/doc/user/project/operations/feature_flags.md b/doc/user/project/operations/feature_flags.md index f13fbcb2722..8716d5feb4a 100644 --- a/doc/user/project/operations/feature_flags.md +++ b/doc/user/project/operations/feature_flags.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Feature Flags **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7433) in GitLab 11.4. @@ -15,6 +21,13 @@ This helps reducing risk and allows you to easily manage which features to enabl GitLab offers a Feature Flags interface that allows you to create, toggle and remove feature flags. +<div class="video-fallback"> + <a href="https://www.youtube.com/watch?v=5tw2p6lwXxo">Watch</a> a use case between Feature Flags and Sentry Error Tracking +</div> +<figure class="video-container"> + <iframe src="https://www.youtube.com/embed/5tw2p6lwXxo" frameborder="0" allowfullscreen="true"> </iframe> +</figure> + ## How it works Underneath, GitLab uses [unleash](https://github.com/Unleash/unleash), a feature @@ -36,10 +49,10 @@ To add a new feature flag: 1. Click on the **New Feature Flag** button. 1. Give it a name. - NOTE: **Note:** - A name can contain only lowercase letters, digits, underscores (`_`) - and dashes (`-`), must start with a letter, and cannot end with a dash (`-`) - or an underscore (`_`). + NOTE: **Note:** + A name can contain only lowercase letters, digits, underscores (`_`) + and dashes (`-`), must start with a letter, and cannot end with a dash (`-`) + or an underscore (`_`). 1. Give it a description (optional, 255 characters max). 1. Define environment [specs](#define-environment-specs). If you want the flag on by default, enable the catch-all [wildcard spec (`*`)](#define-environment-specs) @@ -66,24 +79,59 @@ For example, you may not want to enable a feature flag on production until your first confirmed that the feature is working correctly on testing environments. To handle these situations, you can enable a feature flag on a particular environment -with [Environment specs](../../../ci/environments.md#scoping-environments-with-specs). +with [Environment specs](../../../ci/environments/index.md#scoping-environments-with-specs). You can define multiple specs per flag so that you can control your feature flag more granularly. To define specs for each environment: 1. Navigate to your project's **Operations > Feature Flags**. 1. Click on the **New Feature Flag** button or edit an existing flag. -1. Set the status of the default [spec](../../../ci/environments.md#scoping-environments-with-specs) (`*`). Choose a rollout strategy. This status and rollout strategy combination will be used for _all_ environments. -1. If you want to enable/disable the feature on a specific environment, create a new [spec](../../../ci/environments.md#scoping-environments-with-specs) and type the environment name. +1. Set the status of the default [spec](../../../ci/environments/index.md#scoping-environments-with-specs) (`*`). Choose a rollout strategy. This status and rollout strategy combination will be used for _all_ environments. +1. If you want to enable/disable the feature on a specific environment, create a new [spec](../../../ci/environments/index.md#scoping-environments-with-specs) and type the environment name. 1. Set the status and rollout strategy of the additional spec. This status and rollout strategy combination takes precedence over the default spec since we always use the most specific match available. 1. Click **Create feature flag** or **Update feature flag**.  NOTE: **NOTE** -We'd highly recommend you to use the [Environment](../../../ci/environments.md) +We'd highly recommend you to use the [Environment](../../../ci/environments/index.md) feature in order to quickly assess which flag is enabled per environment. +## Feature flag behavior change in 13.0 + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35555) in GitLab 13.0. + +Starting in GitLab 13.0, you can apply a feature flag strategy across multiple environment specs, +without defining the strategy multiple times. + +This feature is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:feature_flags_new_version) +``` + +To disable it: + +```ruby +Feature.disable(:feature_flags_new_version) +``` + +### Applying a strategy to environments + +After a strategy is defined, it applies to **All Environments** by default. To +make a strategy apply to a specific environment spec: + +1. Click the **Add Environment** button. +1. Create a new + [spec](../../../ci/environments/index.md#scoping-environments-with-specs). + +To apply the strategy to multiple environment specs, repeat these steps. + ## Feature Flag strategies GitLab Feature Flag adopts [Unleash](https://unleash.github.io) @@ -148,12 +196,12 @@ To get the access credentials that your application will need to talk to GitLab: 1. Navigate to your project's **Operations > Feature Flags**. 1. Click on the **Configure** button to see the values: - - **API URL**: URL where the client (application) connects to get a list of feature flags. - - **Instance ID**: Unique token that authorizes the retrieval of the feature flags. - - **Application name**: The name of the running environment. For instance, - if the application runs for production server, application name would be - `production` or similar. This value is used for - [the environment spec evaluation](#define-environment-specs). + - **API URL**: URL where the client (application) connects to get a list of feature flags. + - **Instance ID**: Unique token that authorizes the retrieval of the feature flags. + - **Application name**: The name of the running environment. For instance, + if the application runs for a production server, application name would be + `production` or similar. This value is used for + [the environment spec evaluation](#define-environment-specs). NOTE: **Note:** The meaning of these fields might change over time. For example, we are not sure @@ -231,7 +279,7 @@ func main() { Here's an example of how to integrate the feature flags in a Ruby application. -The Unleash client is given a user id for use with a **Percent rollout (logged in users)** rollout strategy or a list of **Target Users**. +The Unleash client is given a user ID for use with a **Percent rollout (logged in users)** rollout strategy or a list of **Target Users**. ```ruby #!/usr/bin/env ruby @@ -265,3 +313,4 @@ to control them in an automated flow: - [Feature Flags API](../../../api/feature_flags.md) - [Feature Flag Specs API](../../../api/feature_flag_specs.md) +- [Feature Flag User Lists API](../../../api/feature_flag_user_lists.md) diff --git a/doc/user/project/operations/img/alert_detail_v13_0.png b/doc/user/project/operations/img/alert_detail_v13_0.png Binary files differnew file mode 100644 index 00000000000..7da09407cd5 --- /dev/null +++ b/doc/user/project/operations/img/alert_detail_v13_0.png diff --git a/doc/user/project/operations/img/alert_management_1_v13_0.png b/doc/user/project/operations/img/alert_management_1_v13_0.png Binary files differnew file mode 100644 index 00000000000..dbc1e795b16 --- /dev/null +++ b/doc/user/project/operations/img/alert_management_1_v13_0.png diff --git a/doc/user/project/operations/img/alert_management_1_v13_1.png b/doc/user/project/operations/img/alert_management_1_v13_1.png Binary files differnew file mode 100644 index 00000000000..c01b4749eda --- /dev/null +++ b/doc/user/project/operations/img/alert_management_1_v13_1.png diff --git a/doc/user/project/operations/img/alert_management_severity_v13_0.png b/doc/user/project/operations/img/alert_management_severity_v13_0.png Binary files differnew file mode 100644 index 00000000000..f996d6e88f4 --- /dev/null +++ b/doc/user/project/operations/img/alert_management_severity_v13_0.png diff --git a/doc/user/project/operations/index.md b/doc/user/project/operations/index.md index df7ce61525e..954f88fe4f2 100644 --- a/doc/user/project/operations/index.md +++ b/doc/user/project/operations/index.md @@ -4,7 +4,7 @@ GitLab provides a variety of tools to help operate and maintain your applications: - Collect [Prometheus metrics](../integrations/prometheus_library/index.md). -- Deploy to different [environments](../../../ci/environments.md). +- Deploy to different [environments](../../../ci/environments/index.md). - Connect your project to a [Kubernetes cluster](../clusters/index.md). - Manage your infrastructure with [Infrastructure as Code](../../infrastructure/index.md) approaches. - Discover and view errors generated by your applications with [Error Tracking](error_tracking.md). diff --git a/doc/user/project/operations/linking_to_an_external_dashboard.md b/doc/user/project/operations/linking_to_an_external_dashboard.md index 620d9d70e4d..8e1117de4c7 100644 --- a/doc/user/project/operations/linking_to_an_external_dashboard.md +++ b/doc/user/project/operations/linking_to_an_external_dashboard.md @@ -13,7 +13,7 @@ You can add a button to the Monitoring dashboard linking directly to your existi  1. There should now be a button on your - [Monitoring dashboard](../../../ci/environments.md#monitoring-environments) which + [Monitoring dashboard](../../../ci/environments/index.md#monitoring-environments) which will open the URL you entered in the above step.  diff --git a/doc/user/project/operations/tracing.md b/doc/user/project/operations/tracing.md index 8282a980ced..07f60c37f1b 100644 --- a/doc/user/project/operations/tracing.md +++ b/doc/user/project/operations/tracing.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Tracing **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7903) in GitLab Ultimate 11.5. diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md index 172f5d4377f..6e48afad96a 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md @@ -1,5 +1,8 @@ --- type: concepts +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # DNS records overview diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index 07bd2a61eb8..a5fc5b00b53 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -2,6 +2,9 @@ last_updated: 2019-07-04 type: reference, howto disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pages/getting_started_part_three.html' +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Custom domains and SSL/TLS Certificates @@ -60,6 +63,11 @@ according to the type of domain you want to use with your Pages site: - [For subdomains](#for-subdomains), `subdomain.example.com`. - [For both](#for-both-root-and-subdomains). +NOTE: **Note:** +You can [configure IPv6 on self-managed instances](../../../../administration/pages/index.md#advanced-configuration), +but IPv6 is not currently configured for Pages on GitLab.com. +Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/214718) for details. + ##### For root domains Root domains (`example.com`) require: diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md index f80b741fb77..8b180d438ef 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md @@ -1,6 +1,9 @@ --- type: reference description: "Automatic Let's Encrypt SSL certificates for GitLab Pages." +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab Pages integration with Let's Encrypt diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md index bc51a9c90d2..e307b807aaa 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md @@ -1,5 +1,8 @@ --- type: concepts +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # SSL/TLS Certificates diff --git a/doc/user/project/pages/getting_started/fork_sample_project.md b/doc/user/project/pages/getting_started/fork_sample_project.md index ef30606a9ab..00cea846f91 100644 --- a/doc/user/project/pages/getting_started/fork_sample_project.md +++ b/doc/user/project/pages/getting_started/fork_sample_project.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # New Pages website from a forked sample diff --git a/doc/user/project/pages/getting_started/new_or_existing_website.md b/doc/user/project/pages/getting_started/new_or_existing_website.md index 027a238bf02..9a00b724753 100644 --- a/doc/user/project/pages/getting_started/new_or_existing_website.md +++ b/doc/user/project/pages/getting_started/new_or_existing_website.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Start a new Pages website from scratch or deploy an existing website diff --git a/doc/user/project/pages/getting_started/pages_bundled_template.md b/doc/user/project/pages/getting_started/pages_bundled_template.md index 9c097c9acdb..745c50e8d65 100644 --- a/doc/user/project/pages/getting_started/pages_bundled_template.md +++ b/doc/user/project/pages/getting_started/pages_bundled_template.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # New Pages website from a bundled template diff --git a/doc/user/project/pages/getting_started_part_four.md b/doc/user/project/pages/getting_started_part_four.md index 645cc1c5795..4e95b5d5a69 100644 --- a/doc/user/project/pages/getting_started_part_four.md +++ b/doc/user/project/pages/getting_started_part_four.md @@ -1,6 +1,9 @@ --- last_updated: 2020-01-06 type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Creating and Tweaking GitLab CI/CD for GitLab Pages @@ -37,8 +40,8 @@ anything for them to work. Explaining [every detail of GitLab CI/CD](../../../ci/yaml/README.md) and GitLab Runner is out of the scope of this guide, but we'll need to understand just a few things to be able to write our own -`.gitlab-ci.yml` or tweak an existing one. It's an -[Yaml](https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html) file, +`.gitlab-ci.yml` or tweak an existing one. It's a +[YAML](https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html) file, with its own syntax. You can always check your CI syntax with the [GitLab CI/CD Lint Tool](https://gitlab.com/ci/lint). @@ -60,7 +63,7 @@ jekyll build ### Script -To transpose this script to Yaml, it would be like this: +To transpose this script to YAML, it would be like this: ```yaml script: @@ -364,7 +367,7 @@ from Jekyll `_config.yml` file, otherwise Jekyll will understand it as a regular directory to build together with the site: -```yml +```yaml exclude: - vendor ``` diff --git a/doc/user/project/pages/getting_started_part_one.md b/doc/user/project/pages/getting_started_part_one.md index b876e547ba5..c0bdd4b7f0b 100644 --- a/doc/user/project/pages/getting_started_part_one.md +++ b/doc/user/project/pages/getting_started_part_one.md @@ -1,6 +1,9 @@ --- last_updated: 2018-06-04 type: concepts, reference +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab Pages domain names, URLs, and baseurls diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index 02bb8e13f4a..e81c9699153 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -2,6 +2,9 @@ description: 'Learn how to use GitLab Pages to deploy a static website at no additional cost.' last_updated: 2019-06-04 type: index, reference +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab Pages diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index f77c572664a..e36dfd89ab3 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -1,6 +1,9 @@ --- type: reference last_updated: 2020-01-06 +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Exploring GitLab Pages @@ -86,11 +89,9 @@ When using Pages under the general domain of a GitLab instance (`*.example.io`), you _cannot_ use HTTPS with sub-subdomains. That means that if your username/groupname contains a dot, for example `foo.bar`, the domain `https://foo.bar.example.io` will _not_ work. This is a limitation of the -[HTTP Over TLS protocol][rfc]. HTTP pages will continue to work provided you +[HTTP Over TLS protocol](https://tools.ietf.org/html/rfc2818#section-3.1). HTTP pages will continue to work provided you don't redirect HTTP to HTTPS. -[rfc]: https://tools.ietf.org/html/rfc2818#section-3.1 "HTTP Over TLS RFC" - GitLab Pages [does **not** support group websites for subgroups](../../group/subgroups/index.md#limitations). You can only create the highest-level group website. @@ -169,13 +170,10 @@ pages: - pages ``` -See an example that has different files in the [`master` branch][jekyll-master] -and the source files for Jekyll are in a [`pages` branch][jekyll-pages] which +See an example that has different files in the [`master` branch](https://gitlab.com/pages/jekyll-branched/tree/master) +and the source files for Jekyll are in a [`pages` branch](https://gitlab.com/pages/jekyll-branched/tree/pages) which also includes `.gitlab-ci.yml`. -[jekyll-master]: https://gitlab.com/pages/jekyll-branched/tree/master -[jekyll-pages]: https://gitlab.com/pages/jekyll-branched/tree/pages - ### Serving compressed assets Most modern browsers support downloading files in a compressed format. This diff --git a/doc/user/project/pages/pages_access_control.md b/doc/user/project/pages/pages_access_control.md index 1d8119cfe87..7fe4c4c051d 100644 --- a/doc/user/project/pages/pages_access_control.md +++ b/doc/user/project/pages/pages_access_control.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab Pages Access Control @@ -11,6 +14,9 @@ You can enable Pages access control on your project, so that only [members of your project](../../permissions.md#project-members-permissions) (at least Guest) can access your website: +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a demonstration, see [Pages access controls](https://www.youtube.com/watch?v=tSPAr5mQYc8). + 1. Navigate to your project's **Settings > General** and expand **Visibility, project features, permissions**. 1. Toggle the **Pages** button to enable the access control. diff --git a/doc/user/project/protected_tags.md b/doc/user/project/protected_tags.md index 6f68a2c9907..b134d283ba9 100644 --- a/doc/user/project/protected_tags.md +++ b/doc/user/project/protected_tags.md @@ -4,7 +4,7 @@ type: reference, howto # Protected Tags -> [Introduced][ce-10356] in GitLab 9.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10356) in GitLab 9.1. Protected Tags allow control over who has permission to create tags as well as preventing accidental update or deletion once created. Each rule allows you to match either an individual tag name, or use wildcards to control multiple tags at once. @@ -67,5 +67,3 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> - -[ce-10356]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10356 "Protected Tags" diff --git a/doc/user/project/push_options.md b/doc/user/project/push_options.md index e8d94a05e7e..eab88d59867 100644 --- a/doc/user/project/push_options.md +++ b/doc/user/project/push_options.md @@ -33,10 +33,10 @@ git push -o <push_option> You can use push options to skip a CI/CD pipeline, or pass environment variables. -| Push option | Description | -| ------------------------------ | ------------------------------------------------------------------------------------------- | -| `ci.skip` | Do not create a CI pipeline for the latest push. | -| `ci.variable="<name>=<value>"` | Provide [environment variables](../../ci/variables/README.md) to be used in a CI pipeline, if one is created due to the push. | +| Push option | Description | Introduced in version | +| ------------------------------ | ------------------------------------------------------------------------------------------- |---------------------- | +| `ci.skip` | Do not create a CI pipeline for the latest push. | [11.7](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15643) | +| `ci.variable="<name>=<value>"` | Provide [environment variables](../../ci/variables/README.md) to be used in a CI pipeline, if one is created due to the push. | [12.6](https://gitlab.com/gitlab-org/gitlab/issues/27983) | An example of using `ci.skip`: diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 7937b7193d2..e2d0b616e4b 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -42,7 +42,7 @@ The following quick actions are applicable to descriptions, discussions and thre | `/remove_milestone` | ✓ | ✓ | | Remove milestone | | `/label ~label1 ~label2` | ✓ | ✓ | ✓ | Add label(s). Label names can also start without `~` but mixed syntax is not supported | | `/relabel ~label1 ~label2` | ✓ | ✓ | ✓ | Replace existing label(s) with those specified | -| `/unlabel ~label1 ~label2` | ✓ | ✓ | ✓ | Remove all or specific label(s) | +| `/unlabel ~label1 ~label2` or `/remove_label ~label1 ~label2` | ✓ | ✓ | ✓ | Remove all or specific label(s) | | `/copy_metadata <#issue>` | ✓ | ✓ | | Copy labels and milestone from another issue in the project | | `/copy_metadata <!merge_request>` | ✓ | ✓ | | Copy labels and milestone from another merge request in the project | | `/estimate <<W>w <DD>d <hh>h <mm>m>` | ✓ | ✓ | | Set time estimate. For example, `/estimate 1w 3d 2h 14m` | diff --git a/doc/user/project/releases/img/edit_release_page_v13_0.png b/doc/user/project/releases/img/edit_release_page_v13_0.png Binary files differnew file mode 100644 index 00000000000..1b4343019af --- /dev/null +++ b/doc/user/project/releases/img/edit_release_page_v13_0.png diff --git a/doc/user/project/releases/img/release_milestone_dropdown_v13_0.png b/doc/user/project/releases/img/release_milestone_dropdown_v13_0.png Binary files differnew file mode 100644 index 00000000000..453c7ca93cc --- /dev/null +++ b/doc/user/project/releases/img/release_milestone_dropdown_v13_0.png diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index ca28a79abf4..bdb99d16625 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Releases @@ -102,12 +105,15 @@ The physical location of the asset can change at any time and the direct link wi ### Releases associated with milestones -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29020) in GitLab 12.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29020) in GitLab 12.5. +> - [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/39467) to edit milestones in the UI in GitLab 13.0. Releases can optionally be associated with one or more [project milestones](../milestones/index.md#project-milestones-and-group-milestones) by including a `milestones` array in your requests to the -[Releases API](../../../api/releases/index.md#create-a-release). +[Releases API](../../../api/releases/index.md#create-a-release) or by using the dropdown in the [Edit Release](#editing-a-release) page. + + Releases display this association with the **Milestone** indicator in the top section of the Release block on the **Project overview > Releases** page, along @@ -190,22 +196,13 @@ the edit button (pencil icon) in the top-right corner of the release you want to This will bring you to the **Edit Release** page, from which you can change some of the release's details. - + -Currently, it is only possible to edit the release title, notes, and asset -links. To change other release information, such as its tag, associated -milestones, or release date, use the [Releases +Currently, it is only possible to edit the release title, notes, associated milestones, and asset +links. To change other release information, such as its tag, or release date, use the [Releases API](../../../api/releases/index.md#update-a-release). Editing this information through the **Edit Release** page is planned for a future version of GitLab. -Please note that the ability to edit asset links is currently behind a feature -flag which is disabled by default. For self-managed instances, it can be enabled -through the Rails console by a GitLab administrator with the following command: - -```ruby -Feature.enable(:release_asset_link_editing) -``` - ## Notification for Releases > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/26001) in GitLab 12.4. @@ -258,6 +255,9 @@ generate Release Evidence for an existing release. Because of this, [each releas can have multiple Release Evidence snapshots. You can view the Release Evidence and its details on the Release page. +NOTE: **Note:** +When the issue tracker is disabled, release evidence [is not collected](https://gitlab.com/gitlab-org/gitlab/-/issues/208397). + Release Evidence is stored as a JSON object, so you can compare evidence by using commonly-available tools. @@ -360,6 +360,39 @@ terminal. Read the [GitLab Releaser documentation](https://gitlab.com/gitlab-org/gitlab-releaser/-/tree/master/docs/index.md) for details. +## Set a deploy freeze + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29382) in GitLab 13.0. + +With a deploy freeze, you can prevent an unintended production release during a +period of time you specify, whether a company event or public holiday. You can +now rely on the enforcement of policies that are typically outside the scope of +GitLab to reduce uncertainty and risk when automating deployments. + +Deploy freeze periods are set at the Project level, and may be created and +managed using the [Freeze Periods API](../../../api/freeze_periods.md). +Each Freeze Period has a `freeze_start` and a `freeze_end`, which are defined +as [crontab](https://crontab.guru/) entries. If a project contains multiple +freeze periods, all will apply, and should they overlap, the freeze covers the +complete overlapped period. + +During pipeline processing, GitLab CI creates an environment variable named +`$CI_DEPLOY_FREEZE` if the currently executing job is within a +Freeze Period. + +To take advantage of this variable, create a `rules` entry in your +`gitlab-ci.yaml` to prevent the deployment job from executing. + +For example: + +```yaml +deploy_to_production: + stage: deploy + script: deploy_to_prod.sh + rules: + - if: $CI_DEPLOY_FREEZE == null +``` + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/repository/file_finder.md b/doc/user/project/repository/file_finder.md index 91e6d2912d1..ac10071e578 100644 --- a/doc/user/project/repository/file_finder.md +++ b/doc/user/project/repository/file_finder.md @@ -4,7 +4,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/file_finder.html' # File finder -> [Introduced][gh-9889] in GitLab 8.4. +> [Introduced](https://github.com/gitlabhq/gitlabhq/pull/9889) in GitLab 8.4. The file finder feature allows you to search for a file in a repository using the GitLab UI. @@ -12,7 +12,7 @@ GitLab UI. You can find the **Find File** button when in the **Files** section of a project. - + For those who prefer to keep their fingers on the keyboard, there is a [shortcut button](../../shortcuts.md) as well, which you can invoke from _anywhere_ @@ -23,23 +23,20 @@ Press `t` to launch the File search function when in **Issues**, Start typing what you are searching for and watch the magic happen. With the up/down arrows, you go up and down the results, with `Esc` you close the search -and go back to **Files**. +and go back to **Files** ## How it works The File finder feature is powered by the [Fuzzy filter](https://github.com/jeancroy/fuzz-aldrin-plus) library. -It implements a fuzzy search with highlight, and tries to provide intuitive +It implements a fuzzy search with the highlight and tries to provide intuitive results by recognizing patterns that people use while searching. -For example, consider the [GitLab CE repository][ce] and that we want to open +For example, consider the [GitLab FOSS repository](https://gitlab.com/gitlab-org/gitlab-foss/tree/master) and that we want to open the `app/controllers/admin/deploy_keys_controller.rb` file. -Using fuzzy search, we start by typing letters that get us closer to the file. +Using a fuzzy search, we start by typing letters that get us closer to the file. **Tip:** To narrow down your search, include `/` in your search terms. - - -[gh-9889]: https://github.com/gitlabhq/gitlabhq/pull/9889 "File finder pull request" -[ce]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master "GitLab CE repository" + diff --git a/doc/user/project/repository/forking_workflow.md b/doc/user/project/repository/forking_workflow.md index 126144de703..a49701017f3 100644 --- a/doc/user/project/repository/forking_workflow.md +++ b/doc/user/project/repository/forking_workflow.md @@ -72,5 +72,3 @@ changes are added to the repository and branch you're merging into. ## Removing a fork relationship You can unlink your fork from its upstream project in the [advanced settings](../settings/index.md#removing-a-fork-relationship). - -[gitlab flow]: https://about.gitlab.com/blog/2014/09/29/gitlab-flow/ "GitLab Flow blog post" diff --git a/doc/user/project/repository/img/file_finder_find_button.png b/doc/user/project/repository/img/file_finder_find_button.png Binary files differdeleted file mode 100644 index 0c2d7d7bc73..00000000000 --- a/doc/user/project/repository/img/file_finder_find_button.png +++ /dev/null diff --git a/doc/user/project/repository/img/file_finder_find_button_v12_10.png b/doc/user/project/repository/img/file_finder_find_button_v12_10.png Binary files differnew file mode 100644 index 00000000000..e93db946005 --- /dev/null +++ b/doc/user/project/repository/img/file_finder_find_button_v12_10.png diff --git a/doc/user/project/repository/img/file_finder_find_file.png b/doc/user/project/repository/img/file_finder_find_file.png Binary files differdeleted file mode 100644 index c2212c7cd9e..00000000000 --- a/doc/user/project/repository/img/file_finder_find_file.png +++ /dev/null diff --git a/doc/user/project/repository/img/file_finder_find_file_v12_10.png b/doc/user/project/repository/img/file_finder_find_file_v12_10.png Binary files differnew file mode 100644 index 00000000000..1404ccc6d0b --- /dev/null +++ b/doc/user/project/repository/img/file_finder_find_file_v12_10.png diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index f9c953db3e3..055443daa1f 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -162,7 +162,7 @@ Via command line, you can commit multiple times before pushing. you will trigger a pipeline per push, not per commit. - **Skip pipelines:** You can add to you commit message the keyword - [`[ci skip]`](../../../ci/yaml/README.md#skipping-jobs) + [`[ci skip]`](../../../ci/yaml/README.md#skip-pipeline) and GitLab CI/CD will skip that pipeline. - **Cross-link issues and merge requests:** [Cross-linking](../issues/crosslinking_issues.md#from-commit-messages) @@ -199,7 +199,7 @@ of commits to the fewest, and displayed on a nice graph: ## Repository graph -The repository graph displays visually the Git flow strategy used in that repository: +The repository graph displays the history of the repository network visually, including branches and merges. This can help you visualize the Git flow strategy used in the repository:  @@ -215,7 +215,7 @@ minutes.  Not all files are detected, among others; documentation, -vendored code, and most markup languages are excluded. This behaviour can be +vendored code, and most markup languages are excluded. This behavior can be adjusted by overriding the default. For example, to enable `.proto` files to be detected, add the following to `.gitattributes` in the root of your repository. diff --git a/doc/user/project/repository/repository_mirroring.md b/doc/user/project/repository/repository_mirroring.md index 8064eacf404..fdbea385998 100644 --- a/doc/user/project/repository/repository_mirroring.md +++ b/doc/user/project/repository/repository_mirroring.md @@ -55,6 +55,7 @@ For an existing project, you can set up push mirroring as follows: 1. Select **Push** from the **Mirror direction** dropdown. 1. Select an authentication method from the **Authentication method** dropdown, if necessary. 1. Check the **Only mirror protected branches** box, if necessary. +1. Check the **Keep divergent refs** box, if desired. 1. Click the **Mirror repository** button to save the configuration.  @@ -88,6 +89,27 @@ You can choose to only push your protected branches from GitLab to your remote r To use this option, check the **Only mirror protected branches** box when creating a repository mirror. +### Keep divergent refs **(CORE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208828) in GitLab 13.0. + +By default, if any ref on the remote mirror has diverged from the local +repository, the *entire push* will fail, and nothing will be updated. + +For example, if a repository has `master`, `develop`, and `stable` branches that +have been mirrored to a remote, and then a new commit is added to `develop` on +the mirror, the next push attempt will fail, leaving `master` and `stable` +out-of-date despite not having diverged. No change on any branch can be mirrored +until the divergence is resolved. + +With the **Keep divergent refs** option enabled, the `develop` branch is +skipped, allowing `master` and `stable` to be updated. The mirror status will +reflect that `develop` has diverged and was skipped, and be marked as a failed +update. + +NOTE: **Note:** +After the mirror is created, this option can currently only be modified via the [API](../../../api/remote_mirrors.md). + ## Setting up a push mirror from GitLab to GitHub **(CORE)** To set up a mirror from GitLab to GitHub, you need to follow these steps: diff --git a/doc/user/project/repository/x509_signed_commits/index.md b/doc/user/project/repository/x509_signed_commits/index.md index 19238839a5e..20143af0b33 100644 --- a/doc/user/project/repository/x509_signed_commits/index.md +++ b/doc/user/project/repository/x509_signed_commits/index.md @@ -2,21 +2,21 @@ type: concepts, howto --- -# Signing commits with x509 +# Signing commits and tags with X.509 -[x509](https://en.wikipedia.org/wiki/X.509) is a standard format for public key +[X.509](https://en.wikipedia.org/wiki/X.509) is a standard format for public key certificates issued by a public or private Public Key Infrastructure (PKI). -Personal x509 certificates are used for authentication or signing purposes +Personal X.509 certificates are used for authentication or signing purposes such as SMIME, but Git also supports signing of commits and tags -with x509 certificates in a similar way as with [GPG](../gpg_signed_commits/index.md). -The main difference is the trust anchor which is the PKI for x509 certificates +with X.509 certificates in a similar way as with [GPG](../gpg_signed_commits/index.md). +The main difference is the trust anchor which is the PKI for X.509 certificates instead of a web of trust with GPG. -## How GitLab handles x509 +## How GitLab handles X.509 GitLab uses its own certificate store and therefore defines the trust chain. -For a commit to be *verified* by GitLab: +For a commit or tag to be *verified* by GitLab: - The signing certificate email must match a verified email address used by the committer in GitLab. - The Certificate Authority has to be trusted by the GitLab instance, see also @@ -25,9 +25,14 @@ For a commit to be *verified* by GitLab: which is usually up to three years. - The signing time is equal or later then commit time. -NOTE: **Note:** There is no certificate revocation list check in place at the moment. +NOTE: **Note:** Certificate revocation lists are checked on a daily basis via background worker. -## Obtaining an x509 key pair +NOTE: **Note:** Self signed certificates without `authorityKeyIdentifier`, +`subjectKeyIdentifier`, and `crlDistributionPoints` are not supported. We +recommend using certificates from a PKI that are in line with +[RFC 5280](https://tools.ietf.org/html/rfc5280). + +## Obtaining an X.509 key pair If your organization has Public Key Infrastructure (PKI), that PKI will provide an S/MIME key. @@ -37,12 +42,12 @@ own self-signed one, or purchase one. MozillaZine keeps a nice collection of [S/MIME-capable signing authorities](http://kb.mozillazine.org/Getting_an_SMIME_certificate) and some of them generate keys for free. -## Associating your x509 certificate with Git +## Associating your X.509 certificate with Git -To take advantage of X509 signing, you will need Git 2.19.0 or later. You can +To take advantage of X.509 signing, you will need Git 2.19.0 or later. You can check your Git version with: -```sh +```shell git --version ``` @@ -52,7 +57,7 @@ If you have the correct version, you can proceed to configure Git. Configure Git to use your key for signing: -```sh +```shell signingkey = $( gpgsm --list-secret-keys | egrep '(key usage|ID)' | grep -B 1 digitalSignature | awk '/ID/ {print $2}' ) git config --global user.signingkey $signingkey git config --global gpg.format x509 @@ -64,21 +69,21 @@ Install [smimesign](https://github.com/github/smimesign) by downloading the installer or via `brew install smimesign` on MacOS. Get the ID of your certificate with `smimesign --list-keys` and set your -signingkey `git config --global user.signingkey ID`, then configure x509: +signingkey `git config --global user.signingkey ID`, then configure X.509: -```sh +```shell git config --global gpg.x509.program smimesign git config --global gpg.format x509 ``` ## Signing commits -After you have [associated your x509 certificate with Git](#associating-your-x509-certificate-with-git) you +After you have [associated your X.509 certificate with Git](#associating-your-x509-certificate-with-git) you can start signing your commits: 1. Commit like you used to, the only difference is the addition of the `-S` flag: - ```sh + ```shell git commit -S -m "feat: x509 signed commits" ``` @@ -87,7 +92,7 @@ can start signing your commits: If you don't want to type the `-S` flag every time you commit, you can tell Git to sign your commits automatically: -```sh +```shell git config --global commit.gpgsign true ``` @@ -95,6 +100,34 @@ git config --global commit.gpgsign true To verify that a commit is signed, you can use the `--show-signature` flag: -```sh +```shell git log --show-signature ``` + +## Signing tags + +After you have [associated your X.509 certificate with Git](#associating-your-x509-certificate-with-git) you +can start signing your tags: + +1. Tag like you used to, the only difference is the addition of the `-s` flag: + + ```shell + git tag -s v1.1.1 -m "My signed tag" + ``` + +1. Push to GitLab and check that your tags [are verified](#verifying-tags). + +If you don't want to type the `-s` flag every time you tag, you can tell Git +to sign your tags automatically: + +```shell +git config --global tag.gpgsign true +``` + +## Verifying tags + +To verify that a tag is signed, you can use the `--verify` flag: + +```shell +git tag --verify v1.1.1 +``` diff --git a/doc/user/project/requirements/index.md b/doc/user/project/requirements/index.md index 8f4ec7bbbed..50343e52a68 100644 --- a/doc/user/project/requirements/index.md +++ b/doc/user/project/requirements/index.md @@ -10,6 +10,9 @@ Requirements allow you to create criteria to check your products against. They can be based on users, stakeholders, system, software, or anything else you find important to capture. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see [GitLab 12.10 Introduces Requirements Management](https://www.youtube.com/watch?v=uSS7oUNSEoU). +  ## Create a requirement diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index f18a202c63b..d021f259015 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -1,4 +1,4 @@ -# Service Desk **(PREMIUM)** +# Service Desk **(STARTER)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/149) in [GitLab Premium 9.1](https://about.gitlab.com/releases/2017/04/22/gitlab-9-1-released/#service-desk-eep). @@ -95,18 +95,18 @@ directory in your repository. Commit and push to your default branch. The **Thank you email** is the email sent to a user after they submit an issue. The file name of the template has to be `thank_you.md`. -You can use `%{ISSUE_ID}` placeholder which will be replaced by an issue iid in the email and -`%{ISSUE_PATH}` placeholder which will be replaced by project path and the issue iid. +You can use `%{ISSUE_ID}` placeholder which will be replaced by an issue IID in the email and +`%{ISSUE_PATH}` placeholder which will be replaced by project path and the issue IID. As the service desk issues are created as confidential (only project members can see them) -the response email doesn't provide the issue link. +the response email does not provide the issue link. #### New note email The **New note email** is the email sent to a user when the issue they submitted has a new comment. The file name of the template has to be `new_note.md`. -You can use `%{ISSUE_ID}` placeholder which will be replaced by an issue iid +You can use `%{ISSUE_ID}` placeholder which will be replaced by an issue IID in the email, `%{ISSUE_PATH}` placeholder which will be replaced by - project path and the issue iid and `%{NOTE_TEXT}` placeholder which will be replaced by the note text. + project path and the issue IID and `%{NOTE_TEXT}` placeholder which will be replaced by the note text. ### Using custom email display name @@ -115,11 +115,67 @@ in the email, `%{ISSUE_PATH}` placeholder which will be replaced by You can customize the email display name. Emails sent from Service Desk will have this name in the `From` header. The default display name is `GitLab Support Bot`. +### Using custom email address + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2201) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. + +NOTE: **Note:** +This feature is disabled by default. For steps to enable it, see [Enable custom email address](#enable-custom-email-address). + +If the `service_desk_email` feature flag is enabled in your configuration, +then it's possible to create Service Desk issues by sending emails to the +custom Service Desk email address, which should have the following format: +`project_contact+%{key}@example.com`. + +The `%{key}` part is used to find the project where the issue should be created. The +`%{key}` part combines the path to the project and configurable project name suffix: +`<project_full_path>-<project_name_suffix>`. + +You can set the project name suffix in your project's Service Desk settings. +It can contain only lowercase letters (`a-z`), numbers (`0-9`), or underscores (`_`). + + + +For example, suppose you add the following to your configuration: + +```yaml +service_desk_email: + enabled: true + address: "project_contact+%{key}@example.com" + user: "project_support@example.com" + password: "[REDACTED]" + host: "imap.gmail.com" + port: 993 + ssl: true + start_tls: false + log_path: "log/mailroom.log" + mailbox: "inbox" + idle_timeout: 60 + expunge_deleted: true +``` + +In this case, suppose the `mygroup/myproject` project Service Desk settings has the project name +suffix set to `support`, and a user sends an email to `project_contact+mygroup-myproject-support@example.com`. +As a result, a new Service Desk issue is created from this email in the `mygroup/myproject` project. + +#### Enable custom email address + +This feature comes with the `service_desk_email` feature flag disabled by default. +To turn on the feature, ask a GitLab administrator with Rails console access to run the following +command: + +```ruby +Feature.enable(service_desk_email) +``` + +The configuration options are the same as for configuring +[incoming email](../../administration/incoming_email.md#set-it-up). + ## Using Service Desk ### As an end user (issue creator) -To create a Service Desk issue, an end user doesn't need to know anything about +To create a Service Desk issue, an end user does not need to know anything about the GitLab instance. They just send an email to the address they are given, and receive an email back confirming receipt: @@ -136,7 +192,7 @@ And any responses they send will be displayed in the issue itself. ### As a responder to the issue -For responders to the issue, everything works as usual. They'll see a familiar looking +For responders to the issue, everything works as usual. They will see a familiar looking issue tracker, where they can see issues created via customer support requests and filter and interact with them just like other GitLab issues. diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index 7f241a74820..e9521a0567e 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -12,6 +12,7 @@ See also: - [Project import/export API](../../../api/project_import_export.md) - [Project import/export administration Rake tasks](../../../administration/raketasks/project_import_export.md) **(CORE ONLY)** +- [Group import/export](../../group/settings/import_export.md) - [Group import/export API](../../../api/group_import_export.md) To set up a project import/export: @@ -24,6 +25,8 @@ To set up a project import/export: Note the following: +- Imports from a newer version of GitLab are not supported. + The Importing GitLab version must be greater than or equal to the Exporting GitLab version. - Imports will fail unless the import and export GitLab instances are compatible as described in the [Version history](#version-history). - Exports are stored in a temporary [shared directory](../../../development/shared_files.md) @@ -41,11 +44,24 @@ Note the following: ## Version history -The following table lists updates to Import/Export: +Starting with GitLab 13.0, GitLab can import bundles that were exported from a different GitLab deployment. +This ability is limited to two previous GitLab [minor](../../../policy/maintenance.md#versioning) +releases, which is similar to our process for [Security Releases](../../../policy/maintenance.md#security-releases). + +For example: + +| Current version | Can import bundles exported from | +|-----------------|----------------------------------| +| 13.0 | 13.0, 12.10, 12.9 | +| 13.1 | 13.1, 13.0, 12.10 | + +### 12.x + +Prior to 13.0 this was a defined compatibility table: | Exporting GitLab version | Importing GitLab version | | -------------------------- | -------------------------- | -| 11.7 to current | 11.7 to current | +| 11.7 to 13.0 | 11.7 to 13.0 | | 11.1 to 11.6 | 11.1 to 11.6 | | 10.8 to 11.0 | 10.8 to 11.0 | | 10.4 to 10.7 | 10.4 to 10.7 | @@ -66,6 +82,13 @@ Projects can be exported and imported only between versions of GitLab with match For example, 8.10.3 and 8.11 have the same Import/Export version (0.1.3) and the exports between them will be compatible. +## Between CE and EE + +You can export projects from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) and vice versa. +This assumes [version history](#version-history) requirements are met. + +If you're exporting a project from the Enterprise Edition to the Community Edition, you may lose data that is retained only in the Enterprise Edition. For more information, see [downgrading from EE to CE](../../../README.md). + ## Exported contents The following items will be exported: diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index ce94a8f5521..0c98772237b 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -20,6 +20,16 @@ Adjust your project's name, description, avatar, [default branch](../repository/ The project description also partially supports [standard Markdown](../../markdown.md#standard-markdown-and-extensions-in-gitlab). You can use [emphasis](../../markdown.md#emphasis), [links](../../markdown.md#links), and [line-breaks](../../markdown.md#line-breaks) to add more context to the project description. +#### Compliance framework **(ULTIMATE)** + +You can select a framework label to identify that your project has certain compliance requirements or needs additional oversight. Available labels include: + +- GDPR - General Data Protection Regulation +- HIPAA - Health Insurance Portability and Accountability Act +- PCI-DSS - Payment Card Industry-Data Security Standard +- SOC 2 - Service Organization Control 2 +- SOX - Sarbanes-Oxley + ### Sharing and permissions For your repository, you can set up features such as public access, repository features, @@ -46,18 +56,19 @@ Use the switches to enable or disable the following features: | **Forks** | ✓ | Enables [forking](../index.md#fork-a-project) functionality | | **Pipelines** | ✓ | Enables [CI/CD](../../../ci/README.md) functionality | | **Container Registry** | | Activates a [registry](../../packages/container_registry/) for your docker images | -| **Git Large File Storage** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-lfs) | +| **Git Large File Storage** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-large-file-storage-lfs) | | **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration-premium-only) functionality | | **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/) | | **Snippets** | ✓ | Enables [sharing of code and text](../../snippets.md) | | **Pages** | ✓ | Allows you to [publish static websites](../pages/) | +| **Metrics Dashboard** | ✓ | Control access to [metrics dashboard](../integrations/prometheus.md) Some features depend on others: - If you disable the **Issues** option, GitLab also removes the following features: - **Issue Boards** - - [**Service Desk**](#service-desk-premium) **(PREMIUM)** + - [**Service Desk**](#service-desk-starter) **(STARTER)** NOTE: **Note:** When the **Issues** option is disabled, you can still access **Milestones** @@ -70,13 +81,15 @@ Some features depend on others: - If you disable **Repository** functionality, GitLab also disables the following features for your project: - - **Merge Requests** - **Pipelines** - **Container Registry** - **Git Large File Storage** - **Packages** +- Metrics dashboard access requires reading both project environments and deployments. + Users with access to the metrics dashboard can also access environments and deployments. + #### Disabling email notifications Project owners can disable all [email notifications](../../profile/notifications.md#gitlab-notification-emails) @@ -96,7 +109,7 @@ Set up your project's merge request settings:  -### Service Desk **(PREMIUM)** +### Service Desk **(STARTER)** Enable [Service Desk](../service_desk.md) for your project to offer customer support. @@ -247,7 +260,7 @@ To do so: 1. Confirm the action by typing the project's path as instructed. NOTE: **Note:** -Only project maintainers have the [permissions](../../permissions.md#project-members-permissions) +Only project owners have the [permissions](../../permissions.md#project-members-permissions) to remove a fork relationship. ## Operations settings diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md new file mode 100644 index 00000000000..303a6f6d3be --- /dev/null +++ b/doc/user/project/settings/project_access_tokens.md @@ -0,0 +1,55 @@ +# Project access tokens **(CORE ONLY)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2587) in GitLab 13.0. + +Project access tokens are scoped to a project and can be used to authenticate with the [GitLab API](../../../api/README.md#personalproject-access-tokens). + +You can also use project access tokens with Git to authenticate over HTTP or SSH. + +Project access tokens expire on the date you define, at midnight UTC. + +For examples of how you can use a project access token to authenticate with the API, see the following section from our [API Docs](../../../api/README.md#personalproject-access-tokens). + +## Creating a project access token + +1. Log in to GitLab. +1. Navigate to the project you would like to create an access token for. +1. In the **{settings}** **Settings** menu choose **Access Tokens**. +1. Choose a name and optional expiry date for the token. +1. Choose the [desired scopes](#limiting-scopes-of-a-project-access-token). +1. Click the **Create project access token** button. +1. Save the project access token somewhere safe. Once you leave or refresh + the page, you won't be able to access it again. + +## Project bot users + +For each project access token created, a bot user will also be created and added to the project with +["Maintainer" level permissions](../../permissions.md#project-members-permissions). API calls made with a +project access token will be associated to the corresponding bot user. + +These users will appear in **{settings}** **Settings > Members** but can not be modified. +Furthermore, the bot user can not be added to any other project. + +When the project access token is [revoked](#revoking-a-project-access-token) the bot user will be deleted and all +records will be moved to a system-wide user with the username "Ghost User". For more information, +see [Associated Records](../../profile/account/delete_account.md#associated-records). + +## Revoking a project access token + +At any time, you can revoke any project access token by clicking the +respective **Revoke** button in **{settings}** **Settings > Access Tokens**. + +## Limiting scopes of a project access token + +Project access tokens can be created with one or more scopes that allow various +actions that a given token can perform. The available scopes are depicted in +the following table. + +| Scope | Description | +| ------------------ | ----------- | +| `api` | Grants complete read/write access to the scoped project API. | +| `read_api` | Grants read access to the scoped project API. | +| `read_registry` | Allows read-access (pull) to [container registry](../../packages/container_registry/index.md) images if a project is private and authorization is required. | +| `write_registry` | Allows write-access (push) to [container registry](../../packages/container_registry/index.md). | +| `read_repository` | Allows read-only access (pull) to the repository. | +| `write_repository` | Allows read-write access (pull, push) to the repository. | diff --git a/doc/user/project/static_site_editor/img/static_site_editor_v12_10.png b/doc/user/project/static_site_editor/img/static_site_editor_v12_10.png Binary files differdeleted file mode 100644 index 130dff9f30d..00000000000 --- a/doc/user/project/static_site_editor/img/static_site_editor_v12_10.png +++ /dev/null diff --git a/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_0.png b/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_0.png Binary files differnew file mode 100644 index 00000000000..a2f5248bf39 --- /dev/null +++ b/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_0.png diff --git a/doc/user/project/static_site_editor/index.md b/doc/user/project/static_site_editor/index.md index f186ff9919e..e2e498b605a 100644 --- a/doc/user/project/static_site_editor/index.md +++ b/doc/user/project/static_site_editor/index.md @@ -5,7 +5,14 @@ description: "The static site editor enables users to edit content on static web # Static Site Editor -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28758) in GitLab 12.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28758) in GitLab 12.10. +> - WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214559) in GitLab 13.0. + +DANGER: **Danger:** +In GitLab 13.0, we [introduced breaking changes](https://gitlab.com/gitlab-org/gitlab/-/issues/213282) +to the URL structure of the Static Site Editor. Follow the instructions in this +[snippet](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman/snippets/1976539) +to update your project with the latest changes. Static Site Editor enables users to edit content on static websites without prior knowledge of the underlying templating language, site architecture, or @@ -44,7 +51,7 @@ When clicking it, GitLab will open up an editor window from which the content can be directly edited. When you're ready, you can submit your changes in a click of a button: - + When an editor submits their changes, in the background, GitLab automatically creates a new branch, commits their changes, and opens a merge request. The @@ -79,7 +86,8 @@ company and a new feature has been added to the company product. 1. You are assigned the task of updating the documentation. 1. You visit a page and see content that needs to be edited. 1. Click the **Edit this page** button on the production site. -1. The file is opened in the Static Site Editor. +1. The file is opened in the Static Site Editor in **WYSIWYG** mode. If you wish to edit the raw Markdown + instead, you can toggle the **Markdown** mode in the bottom-right corner. 1. You edit the file right there and click **Submit changes**. 1. A new merge request is automatically created and you assign it to your colleague for review. diff --git a/doc/user/project/status_page/index.md b/doc/user/project/status_page/index.md index d292ca94ba9..8ebfb638894 100644 --- a/doc/user/project/status_page/index.md +++ b/doc/user/project/status_page/index.md @@ -1,6 +1,12 @@ -# GitLab Status Page +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2479) in GitLab 12.10. +# GitLab Status Page **(ULTIMATE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2479) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. GitLab Status Page allows you to create and deploy a static website to communicate efficiently to users during an incident. @@ -35,15 +41,15 @@ To use GitLab Status Page you first need to set up your account details for your ### Status Page project -To deploy the status page to AWS S3 you need to add the Status Page project & configure the necessary CI variables. +To deploy the Status Page to AWS S3 you need to add the Status Page project & configure the necessary CI variables. 1. Fork the [Status Page](https://gitlab.com/gitlab-org/status-page) project. This can also be done via [Repository Mirroring](https://gitlab.com/gitlab-org/status-page#repository-mirroring) which will ensure you get the up-to-date Status Page features. 1. Add the following variables in **Settings > CI/CD > Variables**. (To get these variables from Amazon, use your Amazon Console): - - `S3_BUCKET_NAME` - name of the Amazon S3 bucket + - `S3_BUCKET_NAME` - name of the Amazon S3 bucket (If a bucket with the provided name doesn't exist, the first pipeline run will create one and configure it for [static website hosting](https://docs.aws.amazon.com/AmazonS3/latest/dev/HostingWebsiteOnS3Setup.html)) - `AWS_DEFAULT_REGION` - the AWS region - `AWS_ACCESS_KEY_ID` - the AWS access key ID - `AWS_SECRET_ACCESS_KEY` - the AWS secret -1. Run the pipeline to deploy the status page to S3. +1. Run the pipeline to deploy the Status Page to S3. ### Syncing incidents to the Status Page @@ -55,7 +61,7 @@ Once the CI/CD variables are set, you'll need to set up the Project you want to ## Status Page UI -The Status page landing page shows you an overview of the recent incidents. Clicking on an incident will take you to the incident's detail page. +The Status Page landing page shows you an overview of the recent incidents. Clicking on an incident will take you to the incident's detail page.  @@ -64,8 +70,8 @@ The Status page landing page shows you an overview of the recent incidents. Clic The incident detail page shows detailed information about a particular incident. For example: - Status on the incident, including when the incident was last updated. -- The incident title. -- The description of the incident. +- The incident title, including any emojis. +- The description of the incident, including emojis and static images. - A chronological ordered list of updates to the incident.  @@ -76,7 +82,9 @@ The incident detail page shows detailed information about a particular incident. To publish an Incident, you first need to create an issue in the Project you enabled the Status Page settings in. -Once this issue is created, a background worker will publish the issue onto the status page using the credentials you provided during setup. +Once this issue is created, a background worker will publish the issue onto the Status Page using the credentials you provided during setup. +Since all incidents are published publicly, user and group mentions are anonymized with `Incident Responder`, +and titles of non-public [GitLab references](../../markdown.md#special-gitlab-references) are removed. NOTE: **Note:** Confidential issues are not published. If a published issue is made confidential it will be unpublished. @@ -99,4 +107,4 @@ Anyone with access to view the Issue can add an Emoji Award to a comment, so you ### Changing the Incident status -To change the incident status from `open` to `closed`, close the incident issue within GitLab. This will then be updated shortly on the Status page website. +To change the incident status from `open` to `closed`, close the incident issue within GitLab. This will then be updated shortly on the Status Page website. diff --git a/doc/user/project/web_ide/img/admin_clientside_evaluation.png b/doc/user/project/web_ide/img/admin_clientside_evaluation.png Binary files differdeleted file mode 100644 index a930490398b..00000000000 --- a/doc/user/project/web_ide/img/admin_clientside_evaluation.png +++ /dev/null diff --git a/doc/user/project/web_ide/img/admin_live_preview_v13_0.png b/doc/user/project/web_ide/img/admin_live_preview_v13_0.png Binary files differnew file mode 100644 index 00000000000..90129d240bc --- /dev/null +++ b/doc/user/project/web_ide/img/admin_live_preview_v13_0.png diff --git a/doc/user/project/web_ide/img/dark_theme_v13.0.png b/doc/user/project/web_ide/img/dark_theme_v13.0.png Binary files differnew file mode 100644 index 00000000000..6034bc52c05 --- /dev/null +++ b/doc/user/project/web_ide/img/dark_theme_v13.0.png diff --git a/doc/user/project/web_ide/img/clientside_evaluation.png b/doc/user/project/web_ide/img/live_preview_v13_0.png Binary files differindex bd04d3d644b..bd04d3d644b 100644 --- a/doc/user/project/web_ide/img/clientside_evaluation.png +++ b/doc/user/project/web_ide/img/live_preview_v13_0.png diff --git a/doc/user/project/web_ide/img/solarized_light_theme_v13.0.png b/doc/user/project/web_ide/img/solarized_light_theme_v13.0.png Binary files differnew file mode 100644 index 00000000000..f3c4aa142a4 --- /dev/null +++ b/doc/user/project/web_ide/img/solarized_light_theme_v13.0.png diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index c98448ff904..d4daca0e1e4 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -1,6 +1,6 @@ # Web IDE -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4539) in [GitLab Ultimate][ee] 10.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4539) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. > - [Brought to GitLab Core](https://gitlab.com/gitlab-org/gitlab-foss/issues/44157) in 10.7. The Web IDE editor makes it faster and easier to contribute changes to your @@ -15,7 +15,7 @@ and from merge requests. ## File finder -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18323) in [GitLab Core][ce] 10.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18323) in [GitLab Core](https://about.gitlab.com/pricing/) 10.8. The file finder allows you to quickly open files in the current branch by searching. The file finder is launched using the keyboard shortcut `Command-p`, @@ -43,6 +43,20 @@ you can find a more complete list of supported languages in the NOTE: **Note:** Single file editing is based on the [Ace Editor](https://ace.c9.io). +### Themes + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2389) in GitLab 13.0. + +All the themes GitLab supports for syntax highlighting are added to the Web IDE's code editor. +You can pick a theme from your [profile preferences](../../profile/preferences.md). + +The themes are available only in the Web IDE file editor, except for the [dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/209808), +which applies to the entire Web IDE screen. + +| Solarized Light Theme | Dark Theme | +|---------------------------------------------------------------|-----------------------------------------| +|  |  | + ## Commit changes > - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4539) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4 and [brought to GitLab Core](https://gitlab.com/gitlab-org/gitlab-foss/issues/44157) in 10.7. @@ -73,7 +87,7 @@ shows you a preview of the merge request diff if you commit your changes. ## View CI job logs -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19279) in [GitLab Core][ce] 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19279) in [GitLab Core](https://about.gitlab.com/pricing/) 11.0. You can use the Web IDE to quickly fix failing tests by opening the branch or merge request in the Web IDE and opening the logs of the failed @@ -86,7 +100,7 @@ left. ## Switching merge requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19318) in [GitLab Core][ce] 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19318) in [GitLab Core](https://about.gitlab.com/pricing/) 11.0. To switch between your authored and assigned merge requests, click the dropdown in the top of the sidebar to open a list of merge requests. You will @@ -95,34 +109,35 @@ request. ## Switching branches -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20850) in [GitLab Core][ce] 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20850) in [GitLab Core](https://about.gitlab.com/pricing/) 11.2. To switch between branches of the current project repository, click the dropdown in the top of the sidebar to open a list of branches. You will need to commit or discard all your changes before switching to a different branch. -## Client Side Evaluation +## Live Preview -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19764) in [GitLab Core][ce] 11.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19764) in [GitLab Core](https://about.gitlab.com/pricing/) 11.2. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/213853) from _Client Side Evaluation_ to _Live Preview_ in GitLab 13.0. You can use the Web IDE to preview JavaScript projects right in the browser. This feature uses CodeSandbox to compile and bundle the JavaScript used to preview the web application. - + Additionally, for public projects an **Open in CodeSandbox** button is available to transfer the contents of the project into a public CodeSandbox project to quickly share your project with others. -### Enabling Client Side Evaluation +### Enabling Live Preview -The Client Side Evaluation feature needs to be enabled in the GitLab instances -admin settings. Client Side Evaluation is enabled for all projects on +The Live Preview feature needs to be enabled in the GitLab instances +admin settings. Live Preview is enabled for all projects on GitLab.com - + Once you have done that, you can preview projects with a `package.json` file and a `main` entry point inside the Web IDE. An example `package.json` is shown @@ -302,6 +317,3 @@ active terminal at a time. - If the terminal displays **Connection Failure**, then the terminal could not connect to the runner. Please try to stop and restart the terminal. If the problem persists, double check your runner configuration. - -[ce]: https://about.gitlab.com/pricing/ -[ee]: https://about.gitlab.com/pricing/ diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index 88b729272ec..fa3ad4536ef 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -126,7 +126,7 @@ found. The list is ordered alphabetically.  If you have many pages, not all will be listed in the sidebar. Click on -**More pages** to see all of them. +**View All Pages** to see all of them. ## Viewing the history of a wiki page @@ -189,7 +189,24 @@ instructions. On the project's Wiki page, there is a right side navigation that renders the full Wiki pages list by default, with hierarchy. -If the Wiki repository contains a `_sidebar` page, the file of this page replaces the default side navigation. -This custom file serves to render it's custom content, fully replacing the standard sidebar. +To customize the sidebar, you can create a file named `_sidebar` to fully replace the default navigation. + +CAUTION: **Warning:** +Unless you link the `_sidebar` file from your custom nav, to edit it you'll have to access it directly +from the browser's address bar by typing: `https://gitlab.com/<namespace>/<project_name>/-/wikis/_sidebar` (for self-managed GitLab instances, replace `gitlab.com` with your instance's URL). + +Example for `_sidebar` (using Markdown format): + +```markdown +### [Home](home) + +- [Hello World](hello) +- [Foo](foo) +- [Bar](bar) + +--- + +- [Sidebar](_sidebar) +``` Support for displaying a generated TOC with a custom side navigation is planned. diff --git a/doc/user/search/advanced_global_search.md b/doc/user/search/advanced_global_search.md index f38cf8f139e..fac1702f2ac 100644 --- a/doc/user/search/advanced_global_search.md +++ b/doc/user/search/advanced_global_search.md @@ -1,15 +1,17 @@ # Advanced Global Search **(STARTER ONLY)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109) in GitLab [Starter](https://about.gitlab.com/pricing/) 8.4. -> - This is the user documentation. To install and configure Elasticsearch, -> visit the [administrator documentation](../../integration/elasticsearch.md). NOTE: **Note** -Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. We are working on adding it. [Follow this epic for the latest updates](https://gitlab.com/groups/gitlab-org/-/epics/153). +Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. We are working on adding it. +[Follow this epic for the latest updates](https://gitlab.com/groups/gitlab-org/-/epics/153). Leverage Elasticsearch for faster, more advanced code search across your entire GitLab instance. +This is the user documentation. To install and configure Elasticsearch, +visit the [administrator documentation](../../integration/elasticsearch.md). + ## Overview The Advanced Global Search in GitLab is a powerful search service that saves diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index ebb957ad8e2..5113578af9e 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -3,14 +3,16 @@ > **Notes:** > > - Introduced in [GitLab Enterprise Starter](https://about.gitlab.com/pricing/) 9.2 -> - This is the user documentation. To install and configure Elasticsearch, -> visit the [administrator documentation](../../integration/elasticsearch.md). NOTE: **Note** -Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. We are working on adding it. [Follow this epic for the latest updates](https://gitlab.com/groups/gitlab-org/-/epics/153). +Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. We are working on adding it. +[Follow this epic for the latest updates](https://gitlab.com/groups/gitlab-org/-/epics/153). Use advanced queries for more targeted search results. +This is the user documentation. To install and configure Elasticsearch, +visit the [administrator documentation](../../integration/elasticsearch.md). + ## Overview The Advanced Syntax Search is a subset of the diff --git a/doc/user/search/img/filter_approved_by_merge_requests_v13_0.png b/doc/user/search/img/filter_approved_by_merge_requests_v13_0.png Binary files differnew file mode 100644 index 00000000000..ef4c65aea4b --- /dev/null +++ b/doc/user/search/img/filter_approved_by_merge_requests_v13_0.png diff --git a/doc/user/search/index.md b/doc/user/search/index.md index ed2a4b36372..f5efa3519d9 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -41,7 +41,10 @@ groups: - [Label](../project/labels.md) - My-reaction - Confidential - - Epic ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/195704) in GitLab 12.9) + - Epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195704) in GitLab 12.9), + including [child epic](../group/epics/index.md#multi-level-child-epics-ultimate) + ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9029) in + [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0) - Search for this text 1. Select or type the operator to use for filtering the attribute. The following operators are available: @@ -94,10 +97,19 @@ You can filter the **Issues** list to individual instances by their ID. For exam > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/9468) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.9. To filter merge requests by an individual approver, you can type (or select from -the dropdown) `approver` and select the user. +the dropdown) **Approver** and select the user.  +### Filtering merge requests by "approved by" **(STARTER)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30335) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.0. + +To filter merge requests already approved by a specific individual, you can type (or select from +the dropdown) **Approved-By** and select the user. + + + ## Search history You can view recent searches by clicking on the little arrow-clock icon, which is to the left of the search input. Click the search entry to run that search again. This feature is available for issues and merge requests. Searches are stored locally in your browser. diff --git a/doc/user/snippets.md b/doc/user/snippets.md index 0372bb42038..96c8dba11e5 100644 --- a/doc/user/snippets.md +++ b/doc/user/snippets.md @@ -2,7 +2,7 @@ With GitLab Snippets you can store and share bits of code and text with other users. - + Snippets can be maintained using [snippets API](../api/snippets.md). @@ -49,6 +49,67 @@ CAUTION: **Warning:** Make sure to add the file name to get code highlighting and to avoid this [copy-pasting bug](https://gitlab.com/gitlab-org/gitlab/-/issues/22870). +## Versioned Snippets + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/239) in GitLab 13.0. + +Starting in 13.0, snippets (both personal and project snippets) +have version control enabled by default. + +This means that all snippets get their own underlying repository initialized with +a `master` branch at the moment the snippet is created. Whenever a change to the snippet is saved, a +new commit to the master branch is recorded. Commit messages are automatically +generated. The snippet's repository has only one branch (master) by default, deleting +it or creating other branches is not supported. + +Existing snippets will be automatically migrated in 13.0. Their current +content will be saved as the initial commit to the snippets' repository. + +### File names + +Snippets support syntax highlighting based on the filename and +extension provided for them. While it is possible to submit a snippet +without specifying a filename and extension, it needs a valid name so the +content can be created as a file in the snippet's repository. + +In case the user does not attribute a filename and extension to a snippet, +GitLab automatically adds a filename in the format `snippetfile<x>.txt` +where `<x>` represents a number added to the file, starting with 1. This +number increases incrementally when more snippets without an attributed +filename are added. + +When upgrading from an earlier version of GitLab to 13.0, existing snippets +without a supported filename will be renamed to a compatible format. For +example, if the snippet's filename is `http://a-weird-filename.me` it will +be changed to `http-a-weird-filename-me` to be included in the snippet's +repository. As snippets are stored by ID, changing their filenames will not break +direct or embedded links to the snippet. + +### Cloning snippets + +Snippets can be cloned as a regular Git repository using SSH or HTTPS. Click the **Clone** +button above the snippet content to copy the URL of your choice. + + + +This allows you to have a local copy of the snippet's repository and make +changes as needed. You can commit those changes and push them to the remote +master branch. + +### Limitations + +- Binary files are not supported. +- Creating or deleting branches is not supported. Only a default *master*. +branch is used. +- Git tags are not supported in snippet repositories. +- Snippets' repositories are limited to one file. Attempting to push more +than one file will result in an error. +- Revisions are not *yet* visible to the user on the GitLab UI, but +it's planned to be added in future iterations. See the [revisions tab issue](https://gitlab.com/gitlab-org/gitlab/-/issues/39271) +for updates. +- The [maximum size for a snippet](../administration/snippets/index.md#snippets-content-size-limit) +is 50 MB, by default. + ## Discover snippets There are two main ways of how you can discover snippets in GitLab. @@ -87,7 +148,7 @@ snippet was created using the GitLab web interface the original line ending is W > Introduced in GitLab 10.8. Public snippets can not only be shared, but also embedded on any website. This -allows to reuse a GitLab snippet in multiple places and any change to the source +allows us to reuse a GitLab snippet in multiple places and any change to the source is automatically reflected in the embedded snippet. To embed a snippet, first make sure that: @@ -111,6 +172,6 @@ Here's how an embedded snippet looks like: <script src="https://gitlab.com/gitlab-org/gitlab-foss/snippets/1717978.js"></script> -Embedded snippets are displayed with a header that shows the file name if defined, +Embedded snippets are displayed with a header that shows the file name is defined, the snippet size, a link to GitLab, and the actual snippet content. Actions in the header allow users to see the snippet in raw format and download it. diff --git a/doc/user/todos.md b/doc/user/todos.md index d19f1b8f14b..84a7b6afdac 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -38,7 +38,7 @@ A To Do appears on your To-Do List when: - Epic **(ULTIMATE)** - You are `@mentioned` in a comment on a: - Commit - - Design **(PREMIUM)** + - Design - A job in the CI pipeline running for your merge request failed, but this job is not allowed to fail - An open merge request becomes unmergeable due to conflict, and you are either: @@ -137,7 +137,7 @@ There are four kinds of filters you can use on your To-Do List. | Project | Filter by project | | Group | Filter by group | | Author | Filter by the author that triggered the To Do | -| Type | Filter by issue, merge request, or epic **(ULTIMATE)** | +| Type | Filter by issue, merge request, design, or epic **(ULTIMATE)** | | Action | Filter by the action that triggered the To Do | You can also filter by more than one of these at the same time. The possible Actions are |
