diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-18 11:18:50 +0000 |
| commit | 8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781 (patch) | |
| tree | a77e7fe7a93de11213032ed4ab1f33a3db51b738 /doc | |
| parent | 00b35af3db1abfe813a778f643dad221aad51fca (diff) | |
| download | gitlab-ce-8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-1-stable-ee
Diffstat (limited to 'doc')
838 files changed, 26823 insertions, 10704 deletions
diff --git a/doc/.vale/gitlab/Acronyms.yml b/doc/.vale/gitlab/Acronyms.yml new file mode 100644 index 00000000000..5176a18e2b6 --- /dev/null +++ b/doc/.vale/gitlab/Acronyms.yml @@ -0,0 +1,71 @@ +--- +# Warning: gitlab.Acronyms +# +# Checks for unexpanded acronyms. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: conditional +message: '"%s" has no definition.' +link: https://about.gitlab.com/handbook/marketing/growth-marketing/content/editorial-team/#acronyms +level: warning +ignorecase: false +# Ensures that the existence of 'first' implies the existence of 'second'. +first: '\b([A-Z]{3,5})\b' +second: '(?:\b[A-Z][a-z]+ )+\(([A-Z]{3,5})\)' +# ... with the exception of these: +exceptions: + - API + - ARN + - ASCII + - AWS + - CNAME + - CPU + - CSS + - CSV + - DNS + - EKS + - GDK + - GET + - GNU + - GPG + - GPL + - HTML + - HTTP + - HTTPS + - IAM + - IDE + - JSON + - LDAP + - LDAPS + - LESS + - LFS + - NFS + - NGINX + - NOTE + - ONLY + - PGP + - PHP + - POST + - PUT + - RPC + - RSA + - RSS + - SAML + - SCP + - SCSS + - SHA + - SQL + - SSH + - SSL + - SSO + - TIP + - TLS + - TODO + - TOML + - UNIX + - URI + - URL + - VPC + - WIP + - XML + - YAML diff --git a/doc/.vale/gitlab/BadgeCapitalization.yml b/doc/.vale/gitlab/BadgeCapitalization.yml index 7e68a06b4d5..c9e9da3b6ce 100644 --- a/doc/.vale/gitlab/BadgeCapitalization.yml +++ b/doc/.vale/gitlab/BadgeCapitalization.yml @@ -1,5 +1,7 @@ --- -# Verifies that badges are not lower case, which won't render properly. +# Error: gitlab.BadgeCapitalization +# +# Verifies that badges are not mixed case, which won't render properly. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: existence diff --git a/doc/.vale/gitlab/British.yml b/doc/.vale/gitlab/British.yml index 943e85beba1..1e5841d3648 100644 --- a/doc/.vale/gitlab/British.yml +++ b/doc/.vale/gitlab/British.yml @@ -1,9 +1,11 @@ --- -# Checks for use of some of the top misused terms at GitLab. +# Error: gitlab.British +# +# Checks that US spelling is used over British spelling. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: substitution -message: 'Use the American spelling "%s" instead of the British "%s".' +message: 'Use the US spelling "%s" instead of the British "%s".' link: https://about.gitlab.com/handbook/communication/#top-misused-terms level: error ignorecase: true diff --git a/doc/.vale/gitlab/CodeblockFences.yml b/doc/.vale/gitlab/CodeblockFences.yml new file mode 100644 index 00000000000..8b61a1a3c16 --- /dev/null +++ b/doc/.vale/gitlab/CodeblockFences.yml @@ -0,0 +1,13 @@ +--- +# Error: gitlab.CodeblockFences +# +# Ensures all codeblock language tags use the full name, not aliases. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: existence +message: 'Syntax highlighting hint "%s" must be one of: yaml, ruby, plaintext, markdown, javascript, shell, golang, python, dockerfile, or typescript.' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#code-blocks +level: error +scope: raw +raw: + - '\`\`\`(yml|rb|text|md|bash|sh\n|js\n|go\n|py\n|docker\n|ts)' diff --git a/doc/.vale/gitlab/Contractions.yml b/doc/.vale/gitlab/Contractions.yml index f4ec24742da..45212945c53 100644 --- a/doc/.vale/gitlab/Contractions.yml +++ b/doc/.vale/gitlab/Contractions.yml @@ -1,9 +1,11 @@ --- +# Suggestion: gitlab.Contractions +# # Checks for use of common and uncommon contractions. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: substitution -message: Use "%s" instead of "%s", for a friendly, informal tone. +message: 'Use "%s" instead of "%s", for a friendly, informal tone.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language level: suggestion nonword: false diff --git a/doc/.vale/gitlab/CurlStringsQuoted.yml b/doc/.vale/gitlab/CurlStringsQuoted.yml new file mode 100644 index 00000000000..4fcb0069423 --- /dev/null +++ b/doc/.vale/gitlab/CurlStringsQuoted.yml @@ -0,0 +1,13 @@ +--- +# Warning: gitlab.CurlStringsQuoted +# +# Ensures all codeblocks using curl quote any URL strings. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: existence +message: 'Curl commands must wrap URLs in double quotes ("): %s' +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#curl-commands +level: warning +scope: code +raw: + - 'curl.*[^"=]https?://.*' diff --git a/doc/.vale/gitlab/FirstPerson.yml b/doc/.vale/gitlab/FirstPerson.yml index 18c5265b0a6..6db89dd4758 100644 --- a/doc/.vale/gitlab/FirstPerson.yml +++ b/doc/.vale/gitlab/FirstPerson.yml @@ -1,8 +1,11 @@ +--- +# Warning: gitlab.FirstPerson +# # Checks for use of first person pronouns. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: existence -message: '`%s` is a first-person pronoun. Use second- or third-person pronouns (like we, you, us, one) instead.' +message: '"%s" is a first-person pronoun. Use second- or third-person pronouns (like we, you, us, one) instead.' level: warning ignorecase: true link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language diff --git a/doc/.vale/gitlab/InternalLinkExtension.yml b/doc/.vale/gitlab/InternalLinkExtension.yml index d07a2600798..94a935196a7 100644 --- a/doc/.vale/gitlab/InternalLinkExtension.yml +++ b/doc/.vale/gitlab/InternalLinkExtension.yml @@ -1,9 +1,11 @@ --- +# Error: gitlab.InternalLinkExtension +# # Checks that internal links have .md extenstion and not .html extension. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: existence -message: Link %s must use the .md file extension. +message: 'Link "%s" must use the .md file extension.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#links-to-internal-documentation level: error scope: raw diff --git a/doc/.vale/gitlab/LatinTerms.yml b/doc/.vale/gitlab/LatinTerms.yml index 8412631f8fe..a2d024fb1ec 100644 --- a/doc/.vale/gitlab/LatinTerms.yml +++ b/doc/.vale/gitlab/LatinTerms.yml @@ -1,9 +1,11 @@ --- -# Checks for use of latin terms.. +# Warning: gitlab.LatinTerms +# +# Checks for use of latin terms. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: substitution -message: Use "%s" instead of "%s," but consider rewriting the sentence. +message: 'Use "%s" instead of "%s", but consider rewriting the sentence.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language level: warning nonword: true diff --git a/doc/.vale/gitlab/MeaningfulLinkWords.yml b/doc/.vale/gitlab/MeaningfulLinkWords.yml new file mode 100644 index 00000000000..1931112ab3e --- /dev/null +++ b/doc/.vale/gitlab/MeaningfulLinkWords.yml @@ -0,0 +1,15 @@ +--- +# Warning: gitlab.MeaningfulLinkWords +# +# Checks for the presence of semantically unhelpful words in link text. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: existence +message: 'Improve SEO and accessibility by rewriting "%s" in the link text.' +level: warning +scope: link +ignorecase: true +link: https://about.gitlab.com/handbook/communication/#writing-style-guidelines +tokens: + - here + - this page diff --git a/doc/.vale/gitlab/MergeConflictMarkers.yml b/doc/.vale/gitlab/MergeConflictMarkers.yml new file mode 100644 index 00000000000..4d733c856e5 --- /dev/null +++ b/doc/.vale/gitlab/MergeConflictMarkers.yml @@ -0,0 +1,13 @@ +--- +# Error: gitlab.MergeConflictMarkers +# +# Checks for the presence of merge conflict markers. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: existence +message: 'Merge conflict marker "%s" found.' +link: https://docs.gitlab.com/ee/development/code_review.html#merging-a-merge-request +level: error +scope: raw +raw: + - '\n<<<<<<< .+\n|\n=======\n|\n>>>>>>> .+\n' diff --git a/doc/.vale/gitlab/OxfordComma.yml b/doc/.vale/gitlab/OxfordComma.yml index 4b37ba8c2b9..e04d209d960 100644 --- a/doc/.vale/gitlab/OxfordComma.yml +++ b/doc/.vale/gitlab/OxfordComma.yml @@ -1,10 +1,12 @@ --- +# Warning: gitlab.OxfordComma +# # Checks for the lack of an Oxford comma. In some cases, will catch overly # complex sentence structures with lots of commas. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: existence -message: Use a comma before the last "and" or "or" in a list of four or more items. +message: 'Use a comma before the last "and" or "or" in a list of four or more items.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#punctuation level: warning raw: diff --git a/doc/.vale/gitlab/Profanity.yml b/doc/.vale/gitlab/Profanity.yml deleted file mode 100644 index c386b23e52c..00000000000 --- a/doc/.vale/gitlab/Profanity.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -extends: existence -message: "Remove profanity: '%s'" -ignorecase: true -level: error -tokens: - - 'arse(hole)?' - - 'ass(hole)?' - - 'bastard' - - 'bitch' - - 'bloody' - - 'bollocks' - - 'bugger' - - 'cocksucker' - - 'crap' - - 'cunt' - - 'damn' - - 'eff(ing)?' - - 'fart' - - 'fuck(er|ing)?' - - 'goddamn(it?|ed)' - - 'hell' - - 'horseshit' - - 'motherfuck(ers?|ing)' - - 'piss(ing)?' - - 'shit' - - 'tits' - - 'turd' - - 'twat' - - 'wank(er|ing)?' diff --git a/doc/.vale/gitlab/ReferenceLinks.yml b/doc/.vale/gitlab/ReferenceLinks.yml index 35a657710de..8a3b6940187 100644 --- a/doc/.vale/gitlab/ReferenceLinks.yml +++ b/doc/.vale/gitlab/ReferenceLinks.yml @@ -1,3 +1,6 @@ +--- +# Error: gitlab.ReferenceLinks +# # Checks for the presence of reference-style links that must be inline. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ diff --git a/doc/.vale/gitlab/RelativeLinks.yml b/doc/.vale/gitlab/RelativeLinks.yml index 95bd60dd6e4..de24d0608e7 100644 --- a/doc/.vale/gitlab/RelativeLinks.yml +++ b/doc/.vale/gitlab/RelativeLinks.yml @@ -1,9 +1,11 @@ --- +# Error: gitlab.RelativeLinks +# # Checks for the presence of absolute hyperlinks that should be relative. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: existence -message: Link %s must be relative. +message: 'Link "%s" must be relative.' link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#links-to-internal-documentation level: error scope: raw diff --git a/doc/.vale/gitlab/Repetition.yml b/doc/.vale/gitlab/Repetition.yml index 8477a4feb58..76afb7bb5ab 100644 --- a/doc/.vale/gitlab/Repetition.yml +++ b/doc/.vale/gitlab/Repetition.yml @@ -1,3 +1,6 @@ +--- +# Error: gitlab.Repetition +# # Checks for duplicate words, like `the the` or `and and`. # # For a list of all options, see https://errata-ai.github.io/vale/styles/ diff --git a/doc/.vale/gitlab/SentenceLength.yml b/doc/.vale/gitlab/SentenceLength.yml new file mode 100644 index 00000000000..b19b76723a6 --- /dev/null +++ b/doc/.vale/gitlab/SentenceLength.yml @@ -0,0 +1,13 @@ +--- +# Warning: gitlab.SentenceLength +# +# Counts words in a sentence and alerts if a sentence exceeds 25 words. +# +# For a list of all options, see https://errata-ai.github.io/vale/styles/ +extends: occurrence +message: 'Shorter sentences improve readability (max 25 words).' +scope: sentence +link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#language +level: warning +max: 25 +token: \b(\w+)\b diff --git a/doc/.vale/gitlab/SentenceSpacing.yml b/doc/.vale/gitlab/SentenceSpacing.yml index b061f7f6f9e..c460ef3ae65 100644 --- a/doc/.vale/gitlab/SentenceSpacing.yml +++ b/doc/.vale/gitlab/SentenceSpacing.yml @@ -1,4 +1,6 @@ --- +# Error: gitlab.SentenceSpacing +# # Check for the following in common content scenarios: # # - No spaces. diff --git a/doc/.vale/gitlab/Spelling.yml b/doc/.vale/gitlab/Spelling.yml index 9b9b1ef10c2..7bf0f085f5c 100644 --- a/doc/.vale/gitlab/Spelling.yml +++ b/doc/.vale/gitlab/Spelling.yml @@ -1,3 +1,6 @@ +--- +# Warning: gitlab.Spelling +# # Checks for possible spelling mistakes in content, not code. May find false positives # due to links using angle brackets: <https://example.com>. These can be ignored. # diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml index fe690d708ed..8c5f7705417 100644 --- a/doc/.vale/gitlab/SubstitutionWarning.yml +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -1,4 +1,6 @@ --- +# Warning: gitlab.SubstitutionWarning +# # Warns against using common shorthand for terms. # For substitutions flagged as errors, see Substitutions.yml # @@ -10,7 +12,9 @@ level: warning ignorecase: true swap: admin: administrator + blacklist(ed|ing)?: denylist config: configuration distro: distribution info: information repo: repository + whitelist(ed|ing)?: allowlist diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index b9c0dbecf31..156ff3a53f0 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -1,4 +1,6 @@ --- +# Error: gitlab.Substitutions +# # Checks for use of some of the top misused terms at GitLab. # For substitutions only flagged as warnings, see SubstitutionWarning.yml # @@ -9,6 +11,7 @@ link: https://about.gitlab.com/handbook/communication/#top-misused-terms level: error ignorecase: true swap: + frontmatter: front matter GitLabber: GitLab team member gitlab omnibus: Omnibus GitLab param: parameter diff --git a/doc/.vale/gitlab/VersionText.yml b/doc/.vale/gitlab/VersionText.yml index c572a1a926c..9a05103cc39 100644 --- a/doc/.vale/gitlab/VersionText.yml +++ b/doc/.vale/gitlab/VersionText.yml @@ -1,4 +1,6 @@ --- +# Error: gitlab.VersionText +# # Checks that version text is formatted correctly. # # Specifically looks for either of the following that is immediately followed on the next line diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index c5e89f72043..b56fa2861ca 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -1,13 +1,20 @@ Akismet Alertmanager Algolia +Alibaba +allowlist +allowlisting +allowlists +anonymized Ansible Anthos API approvers +architected Artifactory Asana Asciidoctor +Assembla Atlassian Auth0 Authentiq @@ -25,7 +32,10 @@ autoscaler autoscales autoscaling awardable +Axios Azure +B-tree +backfilling backport backported backporting @@ -34,6 +44,7 @@ backtrace backtraced backtraces backtracing +badging Bamboo Bitbucket blockquote @@ -41,6 +52,7 @@ blockquoted blockquotes blockquoting boolean +Bootsnap browsable Bugzilla Buildkite @@ -52,8 +64,16 @@ burndown cacheable CAS CentOS -Chatops +Certbot +chai +chatbot +chatbots +ChatOps +checksummed +checksumming Citrix +Citus +clonable Cloudwatch Cobertura Cognito @@ -63,6 +83,7 @@ compilable composable Conda Consul +Corosync cron crons crontab @@ -72,41 +93,67 @@ crosslinking crosslinks Crossplane CrowdIn +Dangerfile +datetime Debian deduplicate deduplicated deduplicates deduplicating deduplication +deliverables +denylist +denylisting +denylists +deployer +deployers deprovision deprovisioned deprovisioning deprovisions +dequarantine +dequarantined +dequarantining +DevOps discoverability Disqus Dockerfile Dockerfiles +dogfood +dogfoods +dogfooding dotenv downvoted downvotes Dpl Dreamweaver +Ecto Elasticsearch enablement enqueued +enum +enums +ETag +Excon expirable Facebook failover failovers failsafe +fastlane favicon +Filebeat +Fio firewalled Flawfinder Flowdock Fluentd Forgerock +Fugit Gantt Gemnasium +Gemojione +gettext Git Gitaly Gitea @@ -115,6 +162,7 @@ GitLab gitlabsos Gitleaks Gitter +globals Gmail Google Gosec @@ -122,6 +170,7 @@ Gradle Grafana gravatar Gzip +Haml hardcode hardcoded hardcodes @@ -140,6 +189,7 @@ hotfixing http https idempotence +idmapper Ingress initializer initializers @@ -152,39 +202,56 @@ jasmine-jquery JavaScript Jaeger Jenkins +Jenkinsfile Jira jQuery +jsdom JupyterHub kanban kanbans +Kaniko Karma Kerberos Kibana +Kinesis Knative Kramdown +kubectl Kubernetes Kubesec Laravel LDAP +ldapsearch +Leiningen Libravatar Lograge +Logstash lookahead lookaheads lookbehind lookbehinds lookups +loopback Lucene Maildir +Mailgun Makefile Makefiles Markdown markdownlint +matcher +matchers Mattermost mbox +memoization +memoize +memoized +memoizing mergeable Microsoft middleware middlewares +migratus Minikube MinIO mitmproxy @@ -195,16 +262,24 @@ misconfiguration misconfigurations misconfiguring mitigations +mixin +mixins mockup mockups ModSecurity +monorepo +monorepos +mutex nameserver nameservers namespace namespaced namespaces +namespacing +namespacings Nanoc NGINX +Nokogiri npm Nurtch OAuth @@ -213,17 +288,24 @@ offboarded offboarding offboards OmniAuth +onboarding OpenID OpenShift Packagist parallelization parallelizations +passwordless +Patroni performant +phaser +phasers Pipfile Pipfiles Piwik PgBouncer plaintext +Poedit +pooler PostgreSQL precompile preconfigure @@ -233,8 +315,12 @@ prefill prefilled prefilling prefills +preload +preloading +preloads prepend prepended +prepending prepends Pritaly profiler @@ -242,8 +328,8 @@ Prometheus proxied proxies proxying -Pseudonymized -Pseudonymizer +pseudonymized +pseudonymizer Puma Python Qualys @@ -259,20 +345,27 @@ Redcarpet Redis Redmine reCAPTCHA +redirection +redirections +refactorings referer referers +reflog +reflogs reindex reindexed reindexes reindexing relicensing remediations -Repmgr -Repmgrd +repmgr +repmgrd +repurposing requeue requeued requeues reusability +Restlet resynced resyncing resyncs @@ -287,6 +380,8 @@ reverified reverifies reverify Rubix +Rubocop +Rubular runbook runbooks runit @@ -294,11 +389,15 @@ runtime runtimes Salesforce SAML +sandboxing sbt +scatterplot +scatterplots Sendmail Sentry serverless Sidekiq +Sisense sharding shfmt Shibboleth @@ -309,13 +408,18 @@ serializing Sitespeed Slack Slony +smartcard +smartcards SMTP Sobelow +Sourcegraph spidering Splunk SpotBugs SSH +Stackdriver storable +storages strace strikethrough strikethroughs @@ -340,7 +444,9 @@ subquerying substring substrings syslog +tcpdump Tiller +timecop todos tokenizer Tokenizers @@ -352,13 +458,15 @@ tooltips Trello triaging TypeScript +Twilio Twitter Ubuntu unarchive unarchived unarchives -Unassign -Unassigns +unarchiving +unassign +unassigns uncheck unchecked unchecking @@ -380,6 +488,7 @@ unmergeable unmerged unmerges unmerging +unmocked unoptimize unoptimized unoptimizes @@ -393,14 +502,20 @@ unpublished unpublishes unpublishing unreferenced +unregister +unregistered +unregisters +unreplicated unresolve unresolved unresolving unschedule +unscoped unstage unstaged unstages unstaging +unstarted unstash unstashed unstashing @@ -418,6 +533,9 @@ upvotes validator validators vendored +versionless +viewport +viewports virtualized virtualizing Vue @@ -425,14 +543,22 @@ Vuex walkthrough walkthroughs WebdriverIO +Webex webpack webserver whitepaper whitepapers +wireframe +wireframes +wireframed +wireframing Wireshark Wordpress +worktree +worktrees Xcode Xeon YouTrack Zeitwerk Zendesk +zsh diff --git a/doc/README.md b/doc/README.md index c9511b22f8f..fe38b6cb419 100644 --- a/doc/README.md +++ b/doc/README.md @@ -289,7 +289,7 @@ The following documentation relates to the DevOps **Release** stage: | [Environment-specific variables](ci/variables/README.md#limit-the-environment-scopes-of-environment-variables) | Limit the scope of variables to specific environments. | | [GitLab CI/CD](ci/README.md) | Explore the features and capabilities of Continuous Deployment and Delivery with GitLab. | | [GitLab Pages](user/project/pages/index.md) | Build, test, and deploy a static site directly from GitLab. | -| [Protected Runners](ci/runners/README.md#protected-runners) | Select Runners to only pick jobs for protected branches and tags. | +| [Protected Runners](ci/runners/README.md#prevent-runners-from-revealing-sensitive-information) | Select Runners to only pick jobs for protected branches and tags. | | [Scheduled Pipelines](ci/pipelines/schedules.md) | Execute pipelines on a schedule. | <div align="right"> @@ -363,7 +363,7 @@ The following documentation relates to the DevOps **Secure** stage: | Secure Topics | Description | |:------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------| | [Compliance Dashboard](user/compliance/compliance_dashboard/index.md) **(ULTIMATE)** | View the most recent Merge Request activity in a group. | -| [Container Scanning](user/application_security/container_scanning/index.md) **(ULTIMATE)** | Use Clair to scan docker images for known vulnerabilities. | +| [Container Scanning](user/application_security/container_scanning/index.md) **(ULTIMATE)** | Use Clair to scan Docker images for known vulnerabilities. | | [Dependency List](user/application_security/dependency_list/index.md) **(ULTIMATE)** | View your project's dependencies and their known vulnerabilities. | | [Dependency Scanning](user/application_security/dependency_scanning/index.md) **(ULTIMATE)** | Analyze your dependencies for known vulnerabilities. | | [Dynamic Application Security Testing (DAST)](user/application_security/dast/index.md) **(ULTIMATE)** | Analyze running web applications for known vulnerabilities. | diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index e42982e6524..cc94d756f99 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: APM +stage: Manage +group: Analytics info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- @@ -36,7 +36,7 @@ There are two kinds of events logged: ### Impersonation data **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/536) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/536) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. Impersonation is where an administrator uses credentials to perform an action as a different user. @@ -93,13 +93,15 @@ From there, you can see the following actions: - Release was added to a project - Release was updated - Release milestone associations changed -- Permission to approve merge requests by committers was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/issues/7531) in GitLab 12.9) -- Permission to approve merge requests by authors was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/issues/7531) in GitLab 12.9) -- Number of required approvals was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/issues/7531) in GitLab 12.9) +- Permission to approve merge requests by committers was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) +- Permission to approve merge requests by authors was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) +- Number of required approvals was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) + +Project events can also be accessed via the [Project Audit Events API](../api/audit_events.md#project-audit-events-starter) ### Instance events **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2336) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2336) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.3. Server-wide audit logging introduces the ability to observe user actions across the entire instance of your GitLab server, making it easy to understand who @@ -118,10 +120,10 @@ recorded: - Ask for password reset - Grant OAuth access - Started or stopped user impersonation -- Changed username ([introduced](https://gitlab.com/gitlab-org/gitlab/issues/7797) in GitLab 12.8) -- User was deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/issues/251) in GitLab 12.8) -- User was added ([introduced](https://gitlab.com/gitlab-org/gitlab/issues/251) in GitLab 12.8) -- User was blocked via Admin Area ([introduced](https://gitlab.com/gitlab-org/gitlab/issues/251) in GitLab 12.8) +- Changed username ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7797) in GitLab 12.8) +- User was deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) +- User was added ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) +- User was blocked via Admin Area ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) - User was blocked via API ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25872) in GitLab 12.9) It's possible to filter particular actions by choosing an audit data type from @@ -152,7 +154,7 @@ It may make the user interface for your project or audit logs very busy, and the to prevent performance degradations on GitLab instances with very high Git write traffic. In an upcoming release, Audit Logs for Git push events will be enabled -by default. Follow [#7865](https://gitlab.com/gitlab-org/gitlab/issues/7865) for updates. +by default. Follow [#7865](https://gitlab.com/gitlab-org/gitlab/-/issues/7865) for updates. If you still wish to enable **Repository push** events in your instance, follow the steps bellow. diff --git a/doc/administration/auth/README.md b/doc/administration/auth/README.md index f30d6be1775..60e1dfb4637 100644 --- a/doc/administration/auth/README.md +++ b/doc/administration/auth/README.md @@ -1,6 +1,9 @@ --- comments: false type: index +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab authentication and authorization @@ -21,10 +24,9 @@ providers: - [Google](../../integration/google.md) - [JWT](jwt.md) - [Kerberos](../../integration/kerberos.md) -- [LDAP](ldap.md): Includes Active Directory, Apple Open Directory, Open LDAP, +- [LDAP](ldap/index.md): Includes Active Directory, Apple Open Directory, Open LDAP, and 389 Server. - - [LDAP for GitLab EE](ldap-ee.md): LDAP additions to GitLab Enterprise Editions **(STARTER ONLY)** - - [Google Secure LDAP](google_secure_ldap.md) + - [Google Secure LDAP](ldap/google_secure_ldap.md) - [Okta](okta.md) - [Salesforce](../../integration/salesforce.md) - [SAML](../../integration/saml.md) @@ -32,4 +34,6 @@ providers: - [Shibboleth](../../integration/shibboleth.md) - [Smartcard](smartcard.md) **(PREMIUM ONLY)** - [Twitter](../../integration/twitter.md) -- [UltraAuth](../../integration/ultra_auth.md) + +NOTE: **Note:** +UltraAuth has removed their software which supports OmniAuth integration. We have therefore removed all references to UltraAuth integration. diff --git a/doc/administration/auth/authentiq.md b/doc/administration/auth/authentiq.md index e9b32b64160..56f6fddc1af 100644 --- a/doc/administration/auth/authentiq.md +++ b/doc/administration/auth/authentiq.md @@ -1,5 +1,8 @@ --- type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Authentiq OmniAuth Provider diff --git a/doc/administration/auth/cognito.md b/doc/administration/auth/cognito.md index 8d5580ccb6c..b4df6446835 100644 --- a/doc/administration/auth/cognito.md +++ b/doc/administration/auth/cognito.md @@ -1,3 +1,10 @@ +--- +type: concepts, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Amazon Web Services Cognito Amazon Cognito lets you add user sign-up, sign-in, and access control to your GitLab instance. @@ -51,6 +58,8 @@ Include the code block in the `/etc/gitlab/gitlab.rb` file: gitlab_rails['omniauth_providers'] = [ { "name" => "cognito", + # "label" => "Cognito", + # "icon" => nil, # Optional icon URL "app_id" => "CLIENT ID", "app_secret" => "CLIENT SECRET", "args" => { @@ -79,3 +88,5 @@ Include the code block in the `/etc/gitlab/gitlab.rb` file: Your sign-in page should now display a Cognito button below the regular sign-in form. To begin the authentication process, click the icon, and AWS Cognito will ask the user to sign in and authorize the GitLab application. If successful, the user will be redirected and signed in to your GitLab instance. + +For more information, see the [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration). diff --git a/doc/administration/auth/crowd.md b/doc/administration/auth/crowd.md index 71938d4fd2b..254bd259344 100644 --- a/doc/administration/auth/crowd.md +++ b/doc/administration/auth/crowd.md @@ -1,5 +1,8 @@ --- type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Atlassian Crowd OmniAuth Provider diff --git a/doc/administration/auth/google_secure_ldap.md b/doc/administration/auth/google_secure_ldap.md index b643dd2f7b9..d30efc6d3cc 100644 --- a/doc/administration/auth/google_secure_ldap.md +++ b/doc/administration/auth/google_secure_ldap.md @@ -1,219 +1,5 @@ --- -type: reference +redirect_to: 'ldap/google_secure_ldap.md' --- -# Google Secure LDAP **(CORE ONLY)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/46391) in GitLab 11.9. - -[Google Cloud Identity](https://cloud.google.com/identity/) provides a Secure -LDAP service that can be configured with GitLab for authentication and group sync. - -Secure LDAP requires a slightly different configuration than standard LDAP servers. -The steps below cover: - -- Configuring the Secure LDAP Client in the Google Admin console. -- Required GitLab configuration. - -## Configuring Google LDAP client - -1. Navigate to <https://admin.google.com/Dashboard> and sign in as a GSuite domain administrator. - -1. Go to **Apps > LDAP > Add Client**. - -1. Provide an `LDAP client name` and an optional `Description`. Any descriptive - values are acceptable. For example, the name could be 'GitLab' and the - description could be 'GitLab LDAP Client'. Click the **Continue** button. - -  - -1. Set **Access Permission** according to your needs. You must choose either - 'Entire domain (GitLab)' or 'Selected organizational units' for both 'Verify user - credentials' and 'Read user information'. Select 'Add LDAP Client' - - TIP: **Tip:** If you plan to use GitLab [LDAP Group Sync](ldap-ee.md#group-sync) - , turn on 'Read group information'. - -  - -1. Download the generated certificate. This is required for GitLab to - communicate with the Google Secure LDAP service. Save the downloaded certificates - for later use. After downloading, click the **Continue to Client Details** button. - -1. Expand the **Service Status** section and turn the LDAP client 'ON for everyone'. - After selecting 'Save', click on the 'Service Status' bar again to collapse - and return to the rest of the settings. - -1. Expand the **Authentication** section and choose 'Generate New Credentials'. - Copy/note these credentials for later use. After selecting 'Close', click - on the 'Authentication' bar again to collapse and return to the rest of the settings. - -Now the Google Secure LDAP Client configuration is finished. The screenshot below -shows an example of the final settings. Continue on to configure GitLab. - - - -## Configuring GitLab - -Edit GitLab configuration, inserting the access credentials and certificate -obtained earlier. - -The following are the configuration keys that need to be modified using the -values obtained during the LDAP client configuration earlier: - -- `bind_dn`: The access credentials username -- `password`: The access credentials password -- `cert`: The `.crt` file text from the downloaded certificate bundle -- `key`: The `.key` file text from the downloaded certificate bundle - -**For Omnibus installations** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_enabled'] = true - gitlab_rails['ldap_servers'] = YAML.load <<-EOS # remember to close this block with 'EOS' below - main: # 'main' is the GitLab 'provider ID' of this LDAP server - label: 'Google Secure LDAP' - - host: 'ldap.google.com' - port: 636 - uid: 'uid' - bind_dn: 'DizzyHorse' - password: 'd6V5H8nhMUW9AuDP25abXeLd' - encryption: 'simple_tls' - verify_certificates: true - - tls_options: - cert: | - -----BEGIN CERTIFICATE----- - MIIDbDCCAlSgAwIBAgIGAWlzxiIfMA0GCSqGSIb3DQEBCwUAMHcxFDASBgNVBAoTC0dvb2dsZSBJ - bmMuMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UE - CxMGR1N1aXRlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xOTAzMTIyMTE5 - MThaFw0yMjAzMTEyMTE5MThaMHcxFDASBgNVBAoTC0dvb2dsZSBJbmMuMRYwFAYDVQQHEw1Nb3Vu - dGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UECxMGR1N1aXRlMQswCQYDVQQG - EwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB - ALOTy4aC38dyjESk6N8fRsKk8DN23ZX/GaNFL5OUmmA1KWzrvVC881OzNdtGm3vNOIxr9clteEG/ - tQwsmsJvQT5U+GkBt+tGKF/zm7zueHUYqTP7Pg5pxAnAei90qkIRFi17ulObyRHPYv1BbCt8pxNB - 4fG/gAXkFbCNxwh1eiQXXRTfruasCZ4/mHfX7MVm8JmWU9uAVIOLW+DSWOFhrDQduJdGBXJOyC2r - Gqoeg9+tkBmNH/jjxpnEkFW8q7io9DdOUqqNgoidA1h9vpKTs3084sy2DOgUvKN9uXWx14uxIyYU - Y1DnDy0wczcsuRt7l+EgtCEgpsLiLJQbKW+JS1UCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAf60J - yazhbHkDKIH2gFxfm7QLhhnqsmafvl4WP7JqZt0u0KdnvbDPfokdkM87yfbKJU1MTI86M36wEC+1 - P6bzklKz7kXbzAD4GggksAzxsEE64OWHC+Y64Tkxq2NiZTw/76POkcg9StiIXjG0ZcebHub9+Ux/ - rTncip92nDuvgEM7lbPFKRIS/YMhLCk09B/U0F6XLsf1yYjyf5miUTDikPkov23b/YGfpc8kh6hq - 1kqdi6a1cYPP34eAhtRhMqcZU9qezpJF6s9EeN/3YFfKzLODFSsVToBRAdZgGHzj//SAtLyQTD4n - KCSvK1UmaMxNaZyTHg8JnMf0ZuRpv26iSg== - -----END CERTIFICATE----- - - key: | - -----BEGIN PRIVATE KEY----- - MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCzk8uGgt/HcoxEpOjfH0bCpPAz - dt2V/xmjRS+TlJpgNSls671QvPNTszXbRpt7zTiMa/XJbXhBv7UMLJrCb0E+VPhpAbfrRihf85u8 - 7nh1GKkz+z4OacQJwHovdKpCERYte7pTm8kRz2L9QWwrfKcTQeHxv4AF5BWwjccIdXokF10U367m - rAmeP5h31+zFZvCZllPbgFSDi1vg0ljhYaw0HbiXRgVyTsgtqxqqHoPfrZAZjR/448aZxJBVvKu4 - qPQ3TlKqjYKInQNYfb6Sk7N9POLMtgzoFLyjfbl1sdeLsSMmFGNQ5w8tMHM3LLkbe5fhILQhIKbC - 4iyUGylviUtVAgMBAAECggEAIPb0CQy0RJoX+q/lGbRVmnyJpYDf+115WNnl+mrwjdGkeZyqw4v0 - BPzkWYzUFP1esJRO6buBNFybQRFdFW0z5lvVv/zzRKq71aVUBPInxaMRyHuJ8D5lIL8nDtgVOwyE - 7DOGyDtURUMzMjdUwoTe7K+O6QBU4X/1pVPZYgmissYSMmt68LiP8k0p601F4+r5xOi/QEy44aVp - aOJZBUOisKB8BmUXZqmQ4Cy05vU9Xi1rLyzkn9s7fxnZ+JO6Sd1r0Thm1mE0yuPgxkDBh/b4f3/2 - GsQNKKKCiij/6TfkjnBi8ZvWR44LnKpu760g/K7psVNrKwqJG6C/8RAcgISWQQKBgQDop7BaKGhK - 1QMJJ/vnlyYFTucfGLn6bM//pzTys5Gop0tpcfX/Hf6a6Dd+zBhmC3tBmhr80XOX/PiyAIbc0lOI - 31rafZuD/oVx5mlIySWX35EqS14LXmdVs/5vOhsInNgNiE+EPFf1L9YZgG/zA7OUBmqtTeYIPDVC - 7ViJcydItQKBgQDFmK0H0IA6W4opGQo+zQKhefooqZ+RDk9IIZMPOAtnvOM7y3rSVrfsSjzYVuMS - w/RP/vs7rwhaZejnCZ8/7uIqwg4sdUBRzZYR3PRNFeheW+BPZvb+2keRCGzOs7xkbF1mu54qtYTa - HZGZj1OsD83AoMwVLcdLDgO1kw32dkS8IQKBgFRdgoifAHqqVah7VFB9se7Y1tyi5cXWsXI+Wufr - j9U9nQ4GojK52LqpnH4hWnOelDqMvF6TQTyLIk/B+yWWK26Ft/dk9wDdSdystd8L+dLh4k0Y+Whb - +lLMq2YABw+PeJUnqdYE38xsZVHoDjBsVjFGRmbDybeQxauYT7PACy3FAoGBAK2+k9bdNQMbXp7I - j8OszHVkJdz/WXlY1cmdDAxDwXOUGVKIlxTAf7TbiijILZ5gg0Cb+hj+zR9/oI0WXtr+mAv02jWp - W8cSOLS4TnBBpTLjIpdu+BwbnvYeLF6MmEjNKEufCXKQbaLEgTQ/XNlchBSuzwSIXkbWqdhM1+gx - EjtBAoGARAdMIiDMPWIIZg3nNnFebbmtBP0qiBsYohQZ+6i/8s/vautEHBEN6Q0brIU/goo+nTHc - t9VaOkzjCmAJSLPUanuBC8pdYgLu5J20NXUZLD9AE/2bBT3OpezKcdYeI2jqoc1qlWHlNtVtdqQ2 - AcZSFJQjdg5BTyvdEDhaYUKGdRw= - -----END PRIVATE KEY----- - EOS - ``` - -1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. - ---- - -**For installations from source** - -1. Edit `config/gitlab.yml`: - - ```yaml - ldap: - enabled: true - servers: - main: # 'main' is the GitLab 'provider ID' of this LDAP server - label: 'Google Secure LDAP' - - host: 'ldap.google.com' - port: 636 - uid: 'uid' - bind_dn: 'DizzyHorse' - password: 'd6V5H8nhMUW9AuDP25abXeLd' - encryption: 'simple_tls' - verify_certificates: true - - tls_options: - cert: | - -----BEGIN CERTIFICATE----- - MIIDbDCCAlSgAwIBAgIGAWlzxiIfMA0GCSqGSIb3DQEBCwUAMHcxFDASBgNVBAoTC0dvb2dsZSBJ - bmMuMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UE - CxMGR1N1aXRlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xOTAzMTIyMTE5 - MThaFw0yMjAzMTEyMTE5MThaMHcxFDASBgNVBAoTC0dvb2dsZSBJbmMuMRYwFAYDVQQHEw1Nb3Vu - dGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UECxMGR1N1aXRlMQswCQYDVQQG - EwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB - ALOTy4aC38dyjESk6N8fRsKk8DN23ZX/GaNFL5OUmmA1KWzrvVC881OzNdtGm3vNOIxr9clteEG/ - tQwsmsJvQT5U+GkBt+tGKF/zm7zueHUYqTP7Pg5pxAnAei90qkIRFi17ulObyRHPYv1BbCt8pxNB - 4fG/gAXkFbCNxwh1eiQXXRTfruasCZ4/mHfX7MVm8JmWU9uAVIOLW+DSWOFhrDQduJdGBXJOyC2r - Gqoeg9+tkBmNH/jjxpnEkFW8q7io9DdOUqqNgoidA1h9vpKTs3084sy2DOgUvKN9uXWx14uxIyYU - Y1DnDy0wczcsuRt7l+EgtCEgpsLiLJQbKW+JS1UCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAf60J - yazhbHkDKIH2gFxfm7QLhhnqsmafvl4WP7JqZt0u0KdnvbDPfokdkM87yfbKJU1MTI86M36wEC+1 - P6bzklKz7kXbzAD4GggksAzxsEE64OWHC+Y64Tkxq2NiZTw/76POkcg9StiIXjG0ZcebHub9+Ux/ - rTncip92nDuvgEM7lbPFKRIS/YMhLCk09B/U0F6XLsf1yYjyf5miUTDikPkov23b/YGfpc8kh6hq - 1kqdi6a1cYPP34eAhtRhMqcZU9qezpJF6s9EeN/3YFfKzLODFSsVToBRAdZgGHzj//SAtLyQTD4n - KCSvK1UmaMxNaZyTHg8JnMf0ZuRpv26iSg== - -----END CERTIFICATE----- - - key: | - -----BEGIN PRIVATE KEY----- - MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCzk8uGgt/HcoxEpOjfH0bCpPAz - dt2V/xmjRS+TlJpgNSls671QvPNTszXbRpt7zTiMa/XJbXhBv7UMLJrCb0E+VPhpAbfrRihf85u8 - 7nh1GKkz+z4OacQJwHovdKpCERYte7pTm8kRz2L9QWwrfKcTQeHxv4AF5BWwjccIdXokF10U367m - rAmeP5h31+zFZvCZllPbgFSDi1vg0ljhYaw0HbiXRgVyTsgtqxqqHoPfrZAZjR/448aZxJBVvKu4 - qPQ3TlKqjYKInQNYfb6Sk7N9POLMtgzoFLyjfbl1sdeLsSMmFGNQ5w8tMHM3LLkbe5fhILQhIKbC - 4iyUGylviUtVAgMBAAECggEAIPb0CQy0RJoX+q/lGbRVmnyJpYDf+115WNnl+mrwjdGkeZyqw4v0 - BPzkWYzUFP1esJRO6buBNFybQRFdFW0z5lvVv/zzRKq71aVUBPInxaMRyHuJ8D5lIL8nDtgVOwyE - 7DOGyDtURUMzMjdUwoTe7K+O6QBU4X/1pVPZYgmissYSMmt68LiP8k0p601F4+r5xOi/QEy44aVp - aOJZBUOisKB8BmUXZqmQ4Cy05vU9Xi1rLyzkn9s7fxnZ+JO6Sd1r0Thm1mE0yuPgxkDBh/b4f3/2 - GsQNKKKCiij/6TfkjnBi8ZvWR44LnKpu760g/K7psVNrKwqJG6C/8RAcgISWQQKBgQDop7BaKGhK - 1QMJJ/vnlyYFTucfGLn6bM//pzTys5Gop0tpcfX/Hf6a6Dd+zBhmC3tBmhr80XOX/PiyAIbc0lOI - 31rafZuD/oVx5mlIySWX35EqS14LXmdVs/5vOhsInNgNiE+EPFf1L9YZgG/zA7OUBmqtTeYIPDVC - 7ViJcydItQKBgQDFmK0H0IA6W4opGQo+zQKhefooqZ+RDk9IIZMPOAtnvOM7y3rSVrfsSjzYVuMS - w/RP/vs7rwhaZejnCZ8/7uIqwg4sdUBRzZYR3PRNFeheW+BPZvb+2keRCGzOs7xkbF1mu54qtYTa - HZGZj1OsD83AoMwVLcdLDgO1kw32dkS8IQKBgFRdgoifAHqqVah7VFB9se7Y1tyi5cXWsXI+Wufr - j9U9nQ4GojK52LqpnH4hWnOelDqMvF6TQTyLIk/B+yWWK26Ft/dk9wDdSdystd8L+dLh4k0Y+Whb - +lLMq2YABw+PeJUnqdYE38xsZVHoDjBsVjFGRmbDybeQxauYT7PACy3FAoGBAK2+k9bdNQMbXp7I - j8OszHVkJdz/WXlY1cmdDAxDwXOUGVKIlxTAf7TbiijILZ5gg0Cb+hj+zR9/oI0WXtr+mAv02jWp - W8cSOLS4TnBBpTLjIpdu+BwbnvYeLF6MmEjNKEufCXKQbaLEgTQ/XNlchBSuzwSIXkbWqdhM1+gx - EjtBAoGARAdMIiDMPWIIZg3nNnFebbmtBP0qiBsYohQZ+6i/8s/vautEHBEN6Q0brIU/goo+nTHc - t9VaOkzjCmAJSLPUanuBC8pdYgLu5J20NXUZLD9AE/2bBT3OpezKcdYeI2jqoc1qlWHlNtVtdqQ2 - AcZSFJQjdg5BTyvdEDhaYUKGdRw= - -----END PRIVATE KEY----- - ``` - -1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](ldap/google_secure_ldap.md). diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/gitlab_ou.png b/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/gitlab_ou.png Binary files differdeleted file mode 100644 index 223fd0ac401..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/gitlab_ou.png +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/ldap_ou.gif b/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/ldap_ou.gif Binary files differdeleted file mode 100644 index a6727a3d85f..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/ldap_ou.gif +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/user_auth.gif b/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/user_auth.gif Binary files differdeleted file mode 100644 index 36e6085259f..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/img/user_auth.gif +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md b/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md index 01da4528eab..40d021e180c 100644 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md +++ b/doc/administration/auth/how_to_configure_ldap_gitlab_ce/index.md @@ -1,273 +1,5 @@ --- -type: howto +redirect_to: '../ldap/index.md' --- -# How to configure LDAP with GitLab CE - -Managing a large number of users in GitLab can become a burden for system administrators. As an organization grows so do user accounts. Keeping these user accounts in sync across multiple enterprise applications often becomes a time consuming task. - -In this guide we will focus on configuring GitLab with Active Directory. [Active Directory](https://en.wikipedia.org/wiki/Active_Directory) is a popular LDAP compatible directory service provided by Microsoft, included in all modern Windows Server operating systems. - -GitLab has supported LDAP integration since [version 2.2](https://about.gitlab.com/releases/2012/02/22/gitlab-version-2-2/). With GitLab LDAP [group syncing](../how_to_configure_ldap_gitlab_ee/index.md#group-sync) being added to GitLab Enterprise Edition in [version 6.0](https://about.gitlab.com/releases/2013/08/20/gitlab-6-dot-0-released/). LDAP integration has become one of the most popular features in GitLab. - -## Getting started - -### Choosing an LDAP Server - -The main reason organizations choose to utilize a LDAP server is to keep the entire organization's user base consolidated into a central repository. Users can access multiple applications and systems across the IT environment using a single login. Because LDAP is an open, vendor-neutral, industry standard application protocol, the number of applications using LDAP authentication continues to increase. - -There are many commercial and open source [directory servers](https://en.wikipedia.org/wiki/Directory_service#LDAP_implementations) that support the LDAP protocol. Deciding on the right directory server highly depends on the existing IT environment in which the server will be integrated with. - -For example, [Active Directory](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/hh831484(v=ws.11)) is generally favored in a primarily Windows environment, as this allows quick integration with existing services. Other popular directory services include: - -- [Oracle Internet Directory](https://www.oracle.com/middleware/technologies/internet-directory.html) -- [OpenLDAP](https://www.openldap.org/) -- [389 Directory](http://directory.fedoraproject.org/) -- [OpenDJ (Renamed to Forgerock Directory Services)](https://www.forgerock.com/platform/directory-services) -- [ApacheDS](https://directory.apache.org/) - -> GitLab uses the [Net::LDAP](https://rubygems.org/gems/net-ldap) library under the hood. This means it supports all [IETF](https://tools.ietf.org/html/rfc2251) compliant LDAPv3 servers. - -### Active Directory (AD) - -We won't cover the installation and configuration of Windows Server or Active Directory Domain Services in this tutorial. There are a number of resources online to guide you through this process: - -- Install Windows Server 2012 - (`technet.microsoft.com`) - [Installing Windows Server 2012](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/jj134246(v=ws.11)) - -- Install Active Directory Domain Services (AD DS) (`technet.microsoft.com`) - [Install Active Directory Domain Services](https://docs.microsoft.com/en-us/windows-server/identity/ad-ds/deploy/install-active-directory-domain-services--level-100-#BKMK_PS) - -> **Shortcut:** You can quickly install AD DS via PowerShell using -`Install-WindowsFeature AD-Domain-Services -IncludeManagementTools` - -### Creating an AD **OU** structure - -Configuring organizational units (**OU**s) is an important part of setting up Active Directory. **OU**s form the base for an entire organizational structure. Using GitLab as an example we have designed the **OU** structure below using the geographic **OU** model. In the Geographic Model we separate **OU**s for different geographic regions. - -| GitLab **OU** Design | GitLab AD Structure | -| :----------------------------: | :------------------------------: | -|  |  | - -Using PowerShell you can output the **OU** structure as a table (_all names are examples only_): - -```ps -Get-ADObject -LDAPFilter "(objectClass=*)" -SearchBase 'OU=GitLab INT,DC=GitLab,DC=org' -Properties CanonicalName | Format-Table Name,CanonicalName -A -``` - -```plaintext -OU CanonicalName ----- ------------- -GitLab INT GitLab.org/GitLab INT -United States GitLab.org/GitLab INT/United States -Developers GitLab.org/GitLab INT/United States/Developers -Gary Johnson GitLab.org/GitLab INT/United States/Developers/Gary Johnson -Ellis Matthews GitLab.org/GitLab INT/United States/Developers/Ellis Matthews -William Collins GitLab.org/GitLab INT/United States/Developers/William Collins -People Ops GitLab.org/GitLab INT/United States/People Ops -Margaret Baker GitLab.org/GitLab INT/United States/People Ops/Margaret Baker -Libby Hartzler GitLab.org/GitLab INT/United States/People Ops/Libby Hartzler -Victoria Ryles GitLab.org/GitLab INT/United States/People Ops/Victoria Ryles -The Netherlands GitLab.org/GitLab INT/The Netherlands -Developers GitLab.org/GitLab INT/The Netherlands/Developers -John Doe GitLab.org/GitLab INT/The Netherlands/Developers/John Doe -Jon Mealy GitLab.org/GitLab INT/The Netherlands/Developers/Jon Mealy -Jane Weingarten GitLab.org/GitLab INT/The Netherlands/Developers/Jane Weingarten -Production GitLab.org/GitLab INT/The Netherlands/Production -Sarah Konopka GitLab.org/GitLab INT/The Netherlands/Production/Sarah Konopka -Cynthia Bruno GitLab.org/GitLab INT/The Netherlands/Production/Cynthia Bruno -David George GitLab.org/GitLab INT/The Netherlands/Production/David George -United Kingdom GitLab.org/GitLab INT/United Kingdom -Developers GitLab.org/GitLab INT/United Kingdom/Developers -Leroy Fox GitLab.org/GitLab INT/United Kingdom/Developers/Leroy Fox -Christopher Alley GitLab.org/GitLab INT/United Kingdom/Developers/Christopher Alley -Norris Morita GitLab.org/GitLab INT/United Kingdom/Developers/Norris Morita -Support GitLab.org/GitLab INT/United Kingdom/Support -Laura Stanley GitLab.org/GitLab INT/United Kingdom/Support/Laura Stanley -Nikki Schuman GitLab.org/GitLab INT/United Kingdom/Support/Nikki Schuman -Harriet Butcher GitLab.org/GitLab INT/United Kingdom/Support/Harriet Butcher -Global Groups GitLab.org/GitLab INT/Global Groups -DevelopersNL GitLab.org/GitLab INT/Global Groups/DevelopersNL -DevelopersUK GitLab.org/GitLab INT/Global Groups/DevelopersUK -DevelopersUS GitLab.org/GitLab INT/Global Groups/DevelopersUS -ProductionNL GitLab.org/GitLab INT/Global Groups/ProductionNL -SupportUK GitLab.org/GitLab INT/Global Groups/SupportUK -People Ops US GitLab.org/GitLab INT/Global Groups/People Ops US -Global Admins GitLab.org/GitLab INT/Global Groups/Global Admins -``` - -> See [more information](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-powershell-1.0/ff730967(v=technet.10)) on searching Active Directory with Windows PowerShell from [The Scripting Guys](https://devblogs.microsoft.com/scripting/) - -## GitLab LDAP configuration - -The initial configuration of LDAP in GitLab requires changes to the `gitlab.rb` configuration file (`/etc/gitlab/gitlab.rb`). Below is an example of a complete configuration using an Active Directory. - -The two Active Directory specific values are `active_directory: true` and `uid: 'sAMAccountName'`. `sAMAccountName` is an attribute returned by Active Directory used for GitLab usernames. See the example output from `ldapsearch` for a full list of attributes a "person" object (user) has in **AD** - [`ldapsearch` example](#using-ldapsearch-unix) - -> Both group_base and admin_group configuration options are only available in GitLab Enterprise Edition. See [GitLab EE - LDAP Features](../how_to_configure_ldap_gitlab_ee/index.md#gitlab-enterprise-edition---ldap-features) **(STARTER ONLY)** - -### Example `gitlab.rb` LDAP - -```ruby -gitlab_rails['ldap_enabled'] = true -gitlab_rails['ldap_servers'] = { -'main' => { - 'label' => 'GitLab AD', - 'host' => 'ad.example.org', - 'port' => 636, - 'uid' => 'sAMAccountName', - 'encryption' => 'simple_tls', - 'verify_certificates' => true, - 'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=org', - 'password' => 'Password1', - 'active_directory' => true, - 'base' => 'OU=GitLab INT,DC=GitLab,DC=org', - 'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org', - 'admin_group' => 'Global Admins' - } -} -``` - -> **Note:** Remember to run `gitlab-ctl reconfigure` after modifying `gitlab.rb` - -## Security improvements (LDAPS) - -Security is an important aspect when deploying an LDAP server. By default, LDAP traffic is transmitted unsecured. LDAP can be secured using SSL/TLS called LDAPS, or commonly "LDAP over SSL". - -Securing LDAP (enabling LDAPS) on Windows Server 2012 involves installing a valid SSL certificate. For full details see Microsoft's guide [How to enable LDAP over SSL with a third-party certification authority](https://support.microsoft.com/en-us/help/321051/how-to-enable-ldap-over-ssl-with-a-third-party-certification-authority) - -> By default a LDAP service listens for connections on TCP and UDP port 389. LDAPS (LDAP over SSL) listens on port 636 - -### Testing your AD server - -#### Using **AdFind** (Windows) - -You can use the [`AdFind`](https://social.technet.microsoft.com/wiki/contents/articles/7535.adfind-command-examples.aspx) utility (on Windows based systems) to test that your LDAP server is accessible and authentication is working correctly. This is a freeware utility built by [Joe Richards](http://www.joeware.net/freetools/tools/adfind/index.htm). - -**Return all objects** - -You can use the filter `objectclass=*` to return all directory objects. - -```shell -adfind -h ad.example.org:636 -ssl -u "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -up Password1 -b "OU=GitLab INT,DC=GitLab,DC=org" -f (objectClass=*) -``` - -**Return single object using filter** - -You can also retrieve a single object by **specifying** the object name or full **DN**. In this example we specify the object name only `CN=Leroy Fox`. - -```shell -adfind -h ad.example.org:636 -ssl -u "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -up Password1 -b "OU=GitLab INT,DC=GitLab,DC=org" -f (&(objectcategory=person)(CN=Leroy Fox))” -``` - -#### Using **ldapsearch** (Unix) - -You can use the `ldapsearch` utility (on Unix based systems) to test that your LDAP server is accessible and authentication is working correctly. This utility is included in the [`ldap-utils`](https://wiki.debian.org/LDAP/LDAPUtils) package. - -**Return all objects** - -You can use the filter `objectclass=*` to return all directory objects. - -```shell -ldapsearch -D "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" \ --w Password1 -p 636 -h ad.example.org \ --b "OU=GitLab INT,DC=GitLab,DC=org" -Z \ --s sub "(objectclass=*)" -``` - -**Return single object using filter** - -You can also retrieve a single object by **specifying** the object name or full **DN**. In this example we specify the object name only `CN=Leroy Fox`. - -```shell -ldapsearch -D "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -w Password1 -p 389 -h ad.example.org -b "OU=GitLab INT,DC=GitLab,DC=org" -Z -s sub "CN=Leroy Fox" -``` - -**Full output of `ldapsearch` command:** - Filtering for _CN=Leroy Fox_ - -```plaintext -# LDAPv3 -# base <OU=GitLab INT,DC=GitLab,DC=org> with scope subtree -# filter: CN=Leroy Fox -# requesting: ALL -# - -# Leroy Fox, Developers, United Kingdom, GitLab INT, GitLab.org -dn: CN=Leroy Fox,OU=Developers,OU=United Kingdom,OU=GitLab INT,DC=GitLab,DC=or - g -objectClass: top -objectClass: person -objectClass: organizationalPerson -objectClass: user -cn: Leroy Fox -sn: Fox -givenName: Leroy -distinguishedName: CN=Leroy Fox,OU=Developers,OU=United Kingdom,OU=GitLab INT, - DC=GitLab,DC=org -instanceType: 4 -whenCreated: 20170210030500.0Z -whenChanged: 20170213050128.0Z -displayName: Leroy Fox -uSNCreated: 16790 -memberOf: CN=DevelopersUK,OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org -uSNChanged: 20812 -name: Leroy Fox -objectGUID:: rBCAo6NR6E6vfSKgzcUILg== -userAccountControl: 512 -badPwdCount: 0 -codePage: 0 -countryCode: 0 -badPasswordTime: 0 -lastLogoff: 0 -lastLogon: 0 -pwdLastSet: 131311695009850084 -primaryGroupID: 513 -objectSid:: AQUAAAAAAAUVAAAA9GMAb7tdJZvsATf7ZwQAAA== -accountExpires: 9223372036854775807 -logonCount: 0 -sAMAccountName: Leroyf -sAMAccountType: 805306368 -userPrincipalName: Leroyf@GitLab.org -objectCategory: CN=Person,CN=Schema,CN=Configuration,DC=GitLab,DC=org -dSCorePropagationData: 16010101000000.0Z -lastLogonTimestamp: 131314356887754250 - -# search result -search: 2 -result: 0 Success - -# numResponses: 2 -# numEntries: 1 -``` - -## Basic user authentication - -After configuring LDAP, basic authentication will be available. Users can then login using their directory credentials. An extra tab is added to the GitLab login screen for the configured LDAP server (e.g "**GitLab AD**"). - - - -Users that are removed from the LDAP base group (e.g `OU=GitLab INT,DC=GitLab,DC=org`) will be **blocked** in GitLab. [More information](../ldap.md#security) on LDAP security. - -If `allow_username_or_email_login` is enabled in the LDAP configuration, GitLab will ignore everything after the first '@' in the LDAP username used on login. Example: The username `jon.doe@example.com` is converted to `jon.doe` when authenticating with the LDAP server. Disable this setting if you use `userPrincipalName` as the `uid`. - -## LDAP extended features on GitLab EE - -With [GitLab Enterprise Edition (EE)](https://about.gitlab.com/pricing/), besides everything we just described, you'll -have extended functionalities with LDAP, such as: - -- Group sync -- Group permissions -- Updating user permissions -- Multiple LDAP servers - -Read through the article on [LDAP for GitLab EE](../how_to_configure_ldap_gitlab_ee/index.md) **(STARTER ONLY)** for an overview. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](../ldap/index.md). diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/admin_group.png b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/admin_group.png Binary files differdeleted file mode 100644 index 9896379d669..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/admin_group.png +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/group_link_final.png b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/group_link_final.png Binary files differdeleted file mode 100644 index 21fb5a7d0ce..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/group_link_final.png +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/group_linking.gif b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/group_linking.gif Binary files differdeleted file mode 100644 index a0ec2d4f10a..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/group_linking.gif +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/manual_permissions.gif b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/manual_permissions.gif Binary files differdeleted file mode 100644 index a9d5dd7e73e..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/manual_permissions.gif +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/multi_login.gif b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/multi_login.gif Binary files differdeleted file mode 100644 index d317add9837..00000000000 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/img/multi_login.gif +++ /dev/null diff --git a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md index d7b32c8a92a..40d021e180c 100644 --- a/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md +++ b/doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md @@ -1,125 +1,5 @@ --- -type: howto +redirect_to: '../ldap/index.md' --- -# How to configure LDAP with GitLab EE **(STARTER ONLY)** - -This article expands on [How to Configure LDAP with GitLab CE](../how_to_configure_ldap_gitlab_ce/index.md). Make sure to read through it before moving forward. - -## GitLab Enterprise Edition - LDAP features - -[GitLab Enterprise Edition (EE)](https://about.gitlab.com/pricing/) has several advantages when it comes to integrating with Active Directory (LDAP): - -- [Administrator Sync](../ldap-ee.md#administrator-sync): As an extension of group sync, you can automatically manage your global GitLab administrators. Specify a group CN for `admin_group` and all members of the LDAP group will be given administrator privileges. -- [Group Sync](#group-sync): This allows GitLab group membership to be automatically updated based on LDAP group members. -- [Multiple LDAP servers](#multiple-ldap-servers): The ability to configure multiple LDAP servers. This is useful if an organization has different LDAP servers within departments. This is not designed for failover. We're working on [supporting LDAP failover](https://gitlab.com/gitlab-org/gitlab/issues/139) in GitLab. - -- Daily user synchronization: Once a day, GitLab will run a synchronization to check and update GitLab users against LDAP. This process updates all user details automatically. - -In the following section, you'll find a description of each of these features. Read through [LDAP GitLab EE docs](../ldap-ee.md) for complementary information. - - - -All members of the group `Global Admins` will be given **administrator** access to GitLab, allowing them to view the `/admin` dashboard. - -### Group Sync - -Group syncing allows AD (LDAP) groups to be mapped to GitLab groups. This provides more control over per-group user management. To configure group syncing edit the `group_base` **DN** (`'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org'`). This **OU** contains all groups that will be associated with [GitLab groups](../../../user/group/index.md). - -#### Creating group links - example - -As an example, let's suppose we have a "UKGov" GitLab group, which deals with confidential government information. Therefore, users of this group must be given the correct permissions to projects contained within the group. Granular group permissions can be applied based on the AD group. - -**UK Developers** of our "UKGov" group are given **"developer"** permissions. - -_The developer permission allows the development staff to effectively manage all project code, issues, and merge requests._ - -**UK Support** staff of our "UKGov" group are given **"reporter"** permissions. - -_The reporter permission allows support staff to manage issues, labels, and review project code._ - -**US People Ops** of our "UKGov" group are given **"guest"** permissions. - - - -> Guest permissions allows people ops staff to review and lodge new issues while allowing no read or write access to project code or [confidential issues](../../../user/project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues) created by other users. - -See the [permission list](../../../user/permissions.md) for complementary information. - -#### Group permissions - example - -Considering the previous example, our staff will have -access to our GitLab instance with the following structure: - - - -Using this permission structure in our example allows only UK staff access to sensitive information stored in the projects code, while still allowing other teams to work effectively. As all permissions are controlled via AD groups new users can be quickly added to existing groups. New group members will then automatically inherit the required permissions. - -> [More information](../ldap-ee.md#group-sync) on group syncing. - -### Updating user permissions - new feature - -Since GitLab [v8.15](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/822) LDAP user permissions can now be manually overridden by an admin user. To override a user's permissions visit the groups **Members** page and select **Edit permissions**. - - - -### Multiple LDAP servers - -GitLab EE can support multiple LDAP servers. Simply configure another server in the `gitlab.rb` file within the `ldap_servers` block. In the example below we configure a new secondary server with the label **GitLab Secondary AD**. This is shown on the GitLab login screen. Large enterprises often utilize multiple LDAP servers for segregating organizational departments. - - - -Considering the example illustrated on the image above, -our `gitlab.rb` configuration would look like: - -```ruby -gitlab_rails['ldap_enabled'] = true -gitlab_rails['ldap_servers'] = { -'main' => { - 'label' => 'GitLab AD', - 'host' => 'ad.example.org', - 'port' => 636, - 'uid' => 'sAMAccountName', - 'method' => 'ssl', - 'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=org', - 'password' => 'Password1', - 'active_directory' => true, - 'base' => 'OU=GitLab INT,DC=GitLab,DC=org', - 'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org', - 'admin_group' => 'Global Admins' - }, - -'secondary' => { - 'label' => 'GitLab Secondary AD', - 'host' => 'ad-secondary.example.net', - 'port' => 636, - 'uid' => 'sAMAccountName', - 'method' => 'ssl', - 'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=com', - 'password' => 'Password1', - 'active_directory' => true, - 'base' => 'OU=GitLab Secondary,DC=GitLab,DC=com', - 'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=com', - 'admin_group' => 'Global Admins' - } -} -``` - -## Conclusion - -Integration of GitLab with Active Directory (LDAP) reduces the complexity of user management. -It has the advantage of improving user permission controls, while easing the deployment of GitLab into an existing [IT environment](https://www.techopedia.com/definition/29199/it-infrastructure). GitLab EE offers advanced group management and multiple LDAP servers. - -With the assistance of the [GitLab Support](https://about.gitlab.com/support/) team, setting up GitLab with an existing AD/LDAP solution will be a smooth and painless process. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This document was moved to [another location](../ldap/index.md). diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md index 5a773485842..29b192a4845 100644 --- a/doc/administration/auth/jwt.md +++ b/doc/administration/auth/jwt.md @@ -1,5 +1,8 @@ --- type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # JWT OmniAuth provider diff --git a/doc/administration/auth/ldap-ee.md b/doc/administration/auth/ldap-ee.md index 9e193dba08c..f5565628af1 100644 --- a/doc/administration/auth/ldap-ee.md +++ b/doc/administration/auth/ldap-ee.md @@ -1,417 +1,5 @@ --- -type: reference +redirect_to: 'ldap/index.md' --- -# LDAP Additions in GitLab EE **(STARTER ONLY)** - -This section documents LDAP features specific to GitLab Enterprise Edition -[Starter](https://about.gitlab.com/pricing/#self-managed) and above. - -For documentation relevant to both Community Edition and Enterprise Edition, -see the main [LDAP documentation](ldap.md). - -NOTE: **Note:** -[Microsoft Active Directory Trusts](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2008-R2-and-2008/cc771568(v=ws.10)) are not supported - -## Use cases - -- User sync: Once a day, GitLab will update users against LDAP. -- Group sync: Once an hour, GitLab will update group membership - based on LDAP group members. - -## Multiple LDAP servers - -With GitLab Enterprise Edition Starter, you can configure multiple LDAP servers -that your GitLab instance will connect to. - -To add another LDAP server: - -1. Duplicating the settings under [the main configuration](ldap.md#configuration). -1. Edit them to match the additional LDAP server. - -Be sure to choose a different provider ID made of letters a-z and numbers 0-9. -This ID will be stored in the database so that GitLab can remember which LDAP -server a user belongs to. - -## User sync - -Once per day, GitLab will run a worker to check and update GitLab -users against LDAP. - -The process will execute the following access checks: - -- Ensure the user is still present in LDAP. -- If the LDAP server is Active Directory, ensure the user is active (not - blocked/disabled state). This will only be checked if - `active_directory: true` is set in the LDAP configuration. - -NOTE: **Note:** -In Active Directory, a user is marked as disabled/blocked if the user -account control attribute (`userAccountControl:1.2.840.113556.1.4.803`) -has bit 2 set. See <https://ctovswild.com/2009/09/03/bitmask-searches-in-ldap/> -for more information. - -The user will be set to `ldap_blocked` state in GitLab if the above conditions -fail. This means the user will not be able to login or push/pull code. - -The process will also update the following user information: - -- Email address. -- If `sync_ssh_keys` is set, SSH public keys. -- If Kerberos is enabled, Kerberos identity. - -NOTE: **Note:** -The LDAP sync process updates existing users while new users will -be created on first sign in. - -## Group Sync - -If your LDAP supports the `memberof` property, when the user signs in for the -first time GitLab will trigger a sync for groups the user should be a member of. -That way they don't need to wait for the hourly sync to be granted -access to their groups and projects. - -A group sync process will run every hour on the hour, and `group_base` must be set -in LDAP configuration for LDAP synchronizations based on group CN to work. This allows -GitLab group membership to be automatically updated based on LDAP group members. - -The `group_base` configuration should be a base LDAP 'container', such as an -'organization' or 'organizational unit', that contains LDAP groups that should -be available to GitLab. For example, `group_base` could be -`ou=groups,dc=example,dc=com`. In the config file it will look like the -following. - -**Omnibus configuration** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## Base where we can search for groups - ## - ## Ex. ou=groups,dc=gitlab,dc=example - ## - ## - group_base: ou=groups,dc=example,dc=com - EOS - ``` - -1. [Apply your changes to GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -**Source configuration** - -1. Edit `/home/git/gitlab/config/gitlab.yml`: - - ```yaml - production: - ldap: - servers: - main: - # snip... - group_base: ou=groups,dc=example,dc=com - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -To take advantage of group sync, group owners or maintainers will need to [create one -or more LDAP group links](#adding-group-links). - -NOTE: **Note:** -If an LDAP user is a group member when LDAP Synchronization is added, and they are not part of the LDAP group, they will be removed from the group. - -### Adding group links - -Once [group sync has been configured](#group-sync) on the instance, one or more LDAP -groups can be linked to a GitLab group to grant their members access to its -contents. - -Group owners or maintainers can add and use LDAP group links by: - -1. Navigating to the group's **Settings > LDAP Synchronization** page. Here, one or more - LDAP groups and [filters](#filters-premium-only) can be linked to this GitLab group, - each one with a configured [permission level](../../user/permissions.md#group-members-permissions) - for its members. -1. Updating the group's membership by navigating to the group's **Settings > Members** - page and clicking **Sync now**. - -### Filters **(PREMIUM ONLY)** - -In GitLab Premium, you can add an LDAP user filter for group synchronization. -Filters allow for complex logic without creating a special LDAP group. - -To sync GitLab group membership based on an LDAP filter: - -1. Open the **LDAP Synchronization** page for the GitLab group. -1. Select **LDAP user filter** as the **Sync method**. -1. Enter an LDAP user filter in the **LDAP user filter** field. - -The filter must comply with the -syntax defined in [RFC 2254](https://tools.ietf.org/search/rfc2254). - -## Administrator sync - -As an extension of group sync, you can automatically manage your global GitLab -administrators. Specify a group CN for `admin_group` and all members of the -LDAP group will be given administrator privileges. The configuration will look -like the following. - -NOTE: **Note:** -Administrators will not be synced unless `group_base` is also -specified alongside `admin_group`. Also, only specify the CN of the admin -group, as opposed to the full DN. - -**Omnibus configuration** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## Base where we can search for groups - ## - ## Ex. ou=groups,dc=gitlab,dc=example - ## - ## - group_base: ou=groups,dc=example,dc=com - - ## - ## The CN of a group containing GitLab administrators - ## - ## Ex. administrators - ## - ## Note: Not `cn=administrators` or the full DN - ## - ## - admin_group: my_admin_group - - EOS - ``` - -1. [Apply your changes to GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -**Source configuration** - -1. Edit `/home/git/gitlab/config/gitlab.yml`: - - ```yaml - production: - ldap: - servers: - main: - # snip... - group_base: ou=groups,dc=example,dc=com - admin_group: my_admin_group - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## Global group memberships lock - -"Lock memberships to LDAP synchronization" setting allows instance administrators -to lock down user abilities to invite new members to a group. - -When enabled, the following applies: - -- Only administrator can manage memberships of any group including access levels. -- Users are not allowed to share project with other groups or invite members to - a project created in a group. - -## Adjusting LDAP user sync schedule - -> Introduced in GitLab Enterprise Edition Starter. - -NOTE: **Note:** -These are cron formatted values. You can use a crontab generator to create -these values, for example <http://www.crontabgenerator.com/>. - -By default, GitLab will run a worker once per day at 01:30 a.m. server time to -check and update GitLab users against LDAP. - -You can manually configure LDAP user sync times by setting the -following configuration values. The example below shows how to set LDAP user -sync to run once every 12 hours at the top of the hour. - -**Omnibus installations** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -**Source installations** - -1. Edit `config/gitlab.yaml`: - - ```yaml - cron_jobs: - ldap_sync_worker_cron: - "0 */12 * * *" - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## Adjusting LDAP group sync schedule - -NOTE: **Note:** -These are cron formatted values. You can use a crontab generator to create -these values, for example <http://www.crontabgenerator.com/>. - -By default, GitLab will run a group sync process every hour, on the hour. - -CAUTION: **Important:** -It's recommended that you do not run too short intervals as this -could lead to multiple syncs running concurrently. This is primarily a concern -for installations with a large number of LDAP users. Please review the -[LDAP group sync benchmark metrics](#benchmarks) to see how -your installation compares before proceeding. - -You can manually configure LDAP group sync times by setting the -following configuration values. The example below shows how to set group -sync to run once every 2 hours at the top of the hour. - -**Omnibus installations** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -**Source installations** - -1. Edit `config/gitlab.yaml`: - - ```yaml - cron_jobs: - ldap_group_sync_worker_cron: - "*/30 * * * *" - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## External groups - -> Introduced in GitLab Enterprise Edition Starter 8.9. - -Using the `external_groups` setting will allow you to mark all users belonging -to these groups as [external users](../../user/permissions.md#external-users-core-only). -Group membership is checked periodically through the `LdapGroupSync` background -task. - -**Omnibus configuration** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## An array of CNs of groups containing users that should be considered external - ## - ## Ex. ['interns', 'contractors'] - ## - ## Note: Not `cn=interns` or the full DN - ## - external_groups: ['interns', 'contractors'] - EOS - ``` - -1. [Apply your changes to GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -**Source configuration** - -1. Edit `config/gitlab.yaml`: - - ```yaml - production: - ldap: - servers: - main: - # snip... - external_groups: ['interns', 'contractors'] - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## Group sync technical details - -There is a lot going on with group sync 'under the hood'. This section -outlines what LDAP queries are executed and what behavior you can expect -from group sync. - -Group member access will be downgraded from a higher level if their LDAP group -membership changes. For example, if a user has 'Owner' rights in a group and the -next group sync reveals they should only have 'Developer' privileges, their -access will be adjusted accordingly. The only exception is if the user is the -*last* owner in a group. Groups need at least one owner to fulfill -administrative duties. - -### Supported LDAP group types/attributes - -GitLab supports LDAP groups that use member attributes: - -- `member` -- `submember` -- `uniquemember` -- `memberof` -- `memberuid`. - -This means group sync supports, at least, LDAP groups with object class: -`groupOfNames`, `posixGroup`, and `groupOfUniqueNames`. - -Other object classes should work fine as long as members -are defined as one of the mentioned attributes. This also means GitLab supports -Microsoft Active Directory, Apple Open Directory, Open LDAP, and 389 Server. -Other LDAP servers should work, too. - -Active Directory also supports nested groups. Group sync will recursively -resolve membership if `active_directory: true` is set in the configuration file. - -NOTE: **Note:** -Nested group membership will only be resolved if the nested group -also falls within the configured `group_base`. For example, if GitLab sees a -nested group with DN `cn=nested_group,ou=special_groups,dc=example,dc=com` but -the configured `group_base` is `ou=groups,dc=example,dc=com`, `cn=nested_group` -will be ignored. - -### Queries - -- Each LDAP group is queried a maximum of one time with base `group_base` and - filter `(cn=<cn_from_group_link>)`. -- If the LDAP group has the `memberuid` attribute, GitLab will execute another - LDAP query per member to obtain each user's full DN. These queries are - executed with base `base`, scope 'base object', and a filter depending on - whether `user_filter` is set. Filter may be `(uid=<uid_from_group>)` or a - joining of `user_filter`. - -### Benchmarks - -Group sync was written to be as performant as possible. Data is cached, database -queries are optimized, and LDAP queries are minimized. The last benchmark run -revealed the following metrics: - -For 20000 LDAP users, 11000 LDAP groups and 1000 GitLab groups with 10 -LDAP group links each: - -- Initial sync (no existing members assigned in GitLab) took 1.8 hours -- Subsequent syncs (checking membership, no writes) took 15 minutes - -These metrics are meant to provide a baseline and performance may vary based on -any number of factors. This was a pretty extreme benchmark and most instances will -not have near this many users or groups. Disk speed, database performance, -network and LDAP server response time will affect these metrics. - -## Troubleshooting - -Please see our [administrator guide to troubleshooting LDAP](ldap-troubleshooting.md). +This document was moved to [another location](ldap/index.md). diff --git a/doc/administration/auth/ldap-troubleshooting.md b/doc/administration/auth/ldap-troubleshooting.md index 77c9c30d140..a553f449abb 100644 --- a/doc/administration/auth/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap-troubleshooting.md @@ -1,651 +1,5 @@ -# LDAP Troubleshooting for Administrators +--- +redirect_to: 'ldap/ldap-troubleshooting.md' +--- -## Common Problems & Workflows - -### Connection - -#### Connection refused - -If you are getting `Connection Refused` errors when trying to connect to the -LDAP server please double-check the LDAP `port` and `encryption` settings used by -GitLab. Common combinations are `encryption: 'plain'` and `port: 389`, OR -`encryption: 'simple_tls'` and `port: 636`. - -#### Connection times out - -If GitLab cannot reach your LDAP endpoint, you will see a message like this: - -```plaintext -Could not authenticate you from Ldapmain because "Connection timed out - user specified timeout". -``` - -If your configured LDAP provider and/or endpoint is offline or otherwise -unreachable by GitLab, no LDAP user will be able to authenticate and log in. -GitLab does not cache or store credentials for LDAP users to provide authentication -during an LDAP outage. - -Contact your LDAP provider or administrator if you are seeing this error. - -#### Referral error - -If you see `LDAP search error: Referral` in the logs, or when troubleshooting -LDAP Group Sync, this error may indicate a configuration problem. The LDAP -configuration `/etc/gitlab/gitlab.rb` (Omnibus) or `config/gitlab.yml` (source) -is in YAML format and is sensitive to indentation. Check that `group_base` and -`admin_group` configuration keys are indented 2 spaces past the server -identifier. The default identifier is `main` and an example snippet looks like -the following: - -```yaml -main: # 'main' is the GitLab 'provider ID' of this LDAP server - label: 'LDAP' - host: 'ldap.example.com' - ... - group_base: 'cn=my_group,ou=groups,dc=example,dc=com' - admin_group: 'my_admin_group' -``` - -#### Query LDAP **(STARTER ONLY)** - -The following allows you to perform a search in LDAP using the rails console. -Depending on what you're trying to do, it may make more sense to query [a -user](#query-a-user-in-ldap) or [a group](#query-a-group-in-ldap-starter-only) directly, or -even [use `ldapsearch`](#ldapsearch) instead. - -```ruby -adapter = Gitlab::Auth::Ldap::Adapter.new('ldapmain') -options = { - # :base is required - # use .base or .group_base - base: adapter.config.group_base, - - # :filter is optional - # 'cn' looks for all "cn"s under :base - # '*' is the search string - here, it's a wildcard - filter: Net::Ldap::Filter.eq('cn', '*'), - - # :attributes is optional - # the attributes we want to get returned - attributes: %w(dn cn memberuid member submember uniquemember memberof) -} -adapter.ldap_search(options) -``` - -For examples of how this is run, -[review the `Adapter` module](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/ee/gitlab/auth/ldap/adapter.rb). - -### User logins - -#### No users are found - -If [you've confirmed](#ldap-check) that a connection to LDAP can be -established but GitLab doesn't show you LDAP users in the output, one of the -following is most likely true: - -- The `bind_dn` user doesn't have enough permissions to traverse the user tree. -- The user(s) don't fall under the [configured `base`](ldap.md#configuration). -- The [configured `user_filter`](ldap.md#using-an-ldap-filter-to-limit-access-to-your-gitlab-server) blocks access to the user(s). - -In this case, you con confirm which of the above is true using -[ldapsearch](#ldapsearch) with the existing LDAP configuration in your -`/etc/gitlab/gitlab.rb`. - -#### User(s) cannot login - -A user can have trouble logging in for any number of reasons. To get started, -here are some questions to ask yourself: - -- Does the user fall under the [configured `base`](ldap.md#configuration) in - LDAP? The user must fall under this `base` to login. -- Does the user pass through the [configured `user_filter`](ldap.md#using-an-ldap-filter-to-limit-access-to-your-gitlab-server)? - If one is not configured, this question can be ignored. If it is, then the - user must also pass through this filter to be allowed to login. - - Refer to our docs on [debugging the `user_filter`](#debug-ldap-user-filter). - -If the above are both okay, the next place to look for the problem is -the logs themselves while reproducing the issue. - -- Ask the user to login and let it fail. -- [Look through the output](#gitlab-logs) for any errors or other - messages about the login. You may see one of the other error messages on - this page, in which case that section can help resolve the issue. - -If the logs don't lead to the root of the problem, use the -[rails console](#rails-console) to [query this user](#query-a-user-in-ldap) -to see if GitLab can read this user on the LDAP server. - -It can also be helpful to -[debug a user sync](#sync-all-users-starter-only) to -investigate further. - -#### Invalid credentials on login - -If that the login credentials used are accurate on LDAP, ensure the following -are true for the user in question: - -- Make sure the user you are binding with has enough permissions to read the user's - tree and traverse it. -- Check that the `user_filter` is not blocking otherwise valid users. -- Run [an LDAP check command](#ldap-check) to make sure that the LDAP settings - are correct and [GitLab can see your users](#no-users-are-found). - -#### Email has already been taken - -A user tries to login with the correct LDAP credentials, is denied access, -and the [production.log](../logs.md#productionlog) shows an error that looks like this: - -```plaintext -(LDAP) Error saving user <USER DN> (email@example.com): ["Email has already been taken"] -``` - -This error is referring to the email address in LDAP, `email@example.com`. Email -addresses must be unique in GitLab and LDAP links to a user's primary email (as opposed -to any of their possibly-numerous secondary emails). Another user (or even the -same user) has the email `email@example.com` set as a secondary email, which -is throwing this error. - -We can check where this conflicting email address is coming from using the -[rails console](#rails-console). Once in the console, run the following: - -```ruby -# This searches for an email among the primary AND secondary emails -user = User.find_by_any_email('email@example.com') -user.username -``` - -This will show you which user has this email address. One of two steps will -have to be taken here: - -- To create a new GitLab user/username for this user when logging in with LDAP, - remove the secondary email to remove the conflict. -- To use an existing GitLab user/username for this user to use with LDAP, - remove this email as a secondary email and make it a primary one so GitLab - will associate this profile to the LDAP identity. - -The user can do either of these steps [in their -profile](../../user/profile/index.md#user-profile) or an admin can do it. - -#### Debug LDAP user filter - -[`ldapsearch`](#ldapsearch) allows you to test your configured -[user filter](ldap.md#using-an-ldap-filter-to-limit-access-to-your-gitlab-server) -to confirm that it returns the users you expect it to return. - -```shell -ldapsearch -H ldaps://$host:$port -D "$bind_dn" -y bind_dn_password.txt -b "$base" "$user_filter" sAMAccountName -``` - -- Variables beginning with a `$` refer to a variable from the LDAP section of - your configuration file. -- Replace `ldaps://` with `ldap://` if you are using the plain authentication method. - Port `389` is the default `ldap://` port and `636` is the default `ldaps://` - port. -- We are assuming the password for the `bind_dn` user is in `bind_dn_password.txt`. - -#### Sync all users **(STARTER ONLY)** - -The output from a manual [user sync](ldap-ee.md#user-sync) can show you what happens when -GitLab tries to sync its users against LDAP. Enter the [rails console](#rails-console) -and then run: - -```ruby -Rails.logger.level = Logger::DEBUG - -LdapSyncWorker.new.perform -``` - -Next, [learn how to read the -output](#example-console-output-after-a-user-sync-starter-only). - -##### Example console output after a user sync **(STARTER ONLY)** - -The output from a [manual user sync](#sync-all-users-starter-only) will be very verbose, and a -single user's successful sync can look like this: - -```shell -Syncing user John, email@example.com - Identity Load (0.9ms) SELECT "identities".* FROM "identities" WHERE "identities"."user_id" = 20 AND (provider LIKE 'ldap%') LIMIT 1 -Instantiating Gitlab::Auth::Ldap::Person with LDIF: -dn: cn=John Smith,ou=people,dc=example,dc=com -cn: John Smith -mail: email@example.com -memberof: cn=admin_staff,ou=people,dc=example,dc=com -uid: John - - UserSyncedAttributesMetadata Load (0.9ms) SELECT "user_synced_attributes_metadata".* FROM "user_synced_attributes_metadata" WHERE "user_synced_attributes_metadata"."user_id" = 20 LIMIT 1 - (0.3ms) BEGIN - Namespace Load (1.0ms) SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."owner_id" = 20 AND "namespaces"."type" IS NULL LIMIT 1 - Route Load (0.8ms) SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 27 AND "routes"."source_type" = 'Namespace' LIMIT 1 - Ci::Runner Load (1.1ms) SELECT "ci_runners".* FROM "ci_runners" INNER JOIN "ci_runner_namespaces" ON "ci_runners"."id" = "ci_runner_namespaces"."runner_id" WHERE "ci_runner_namespaces"."namespace_id" = 27 - (0.7ms) COMMIT - (0.4ms) BEGIN - Route Load (0.8ms) SELECT "routes".* FROM "routes" WHERE (LOWER("routes"."path") = LOWER('John')) - Namespace Load (1.0ms) SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" = 27 LIMIT 1 - Route Exists (0.9ms) SELECT 1 AS one FROM "routes" WHERE LOWER("routes"."path") = LOWER('John') AND "routes"."id" != 50 LIMIT 1 - User Update (1.1ms) UPDATE "users" SET "updated_at" = '2019-10-17 14:40:59.751685', "last_credential_check_at" = '2019-10-17 14:40:59.738714' WHERE "users"."id" = 20 -``` - -There's a lot here, so let's go over what could be helpful when debugging. - -First, GitLab will look for all users that have previously -logged in with LDAP and iterate on them. Each user's sync will start with -the following line that contains the user's username and email, as they -exist in GitLab now: - -```shell -Syncing user John, email@example.com -``` - -If you don't find a particular user's GitLab email in the output, then that -user hasn't logged in with LDAP yet. - -Next, GitLab searches its `identities` table for the existing -link between this user and the configured LDAP provider(s): - -```sql - Identity Load (0.9ms) SELECT "identities".* FROM "identities" WHERE "identities"."user_id" = 20 AND (provider LIKE 'ldap%') LIMIT 1 -``` - -The identity object will have the DN that GitLab will use to look for the user -in LDAP. If the DN isn't found, the email is used instead. We can see that -this user is found in LDAP: - -```shell -Instantiating Gitlab::Auth::Ldap::Person with LDIF: -dn: cn=John Smith,ou=people,dc=example,dc=com -cn: John Smith -mail: email@example.com -memberof: cn=admin_staff,ou=people,dc=example,dc=com -uid: John -``` - -If the user wasn't found in LDAP with either the DN or email, you may see the -following message instead: - -```shell -LDAP search error: No Such Object -``` - -...in which case the user will be blocked: - -```shell - User Update (0.4ms) UPDATE "users" SET "state" = $1, "updated_at" = $2 WHERE "users"."id" = $3 [["state", "ldap_blocked"], ["updated_at", "2019-10-18 15:46:22.902177"], ["id", 20]] -``` - -Once the user is found in LDAP the rest of the output will update the GitLab -database with any changes. - -#### Query a user in LDAP - -This will test that GitLab can reach out to LDAP and read a particular user. -It can expose potential errors connecting to and/or querying LDAP -that may seem to fail silently in the GitLab UI. - -```ruby -Rails.logger.level = Logger::DEBUG - -adapter = Gitlab::Auth::Ldap::Adapter.new('ldapmain') # If `main` is the LDAP provider -Gitlab::Auth::Ldap::Person.find_by_uid('<uid>', adapter) -``` - -### Group memberships **(STARTER ONLY)** - -#### Membership(s) not granted **(STARTER ONLY)** - -Sometimes you may think a particular user should be added to a GitLab group via -LDAP group sync, but for some reason it's not happening. There are several -things to check to debug the situation. - -- Ensure LDAP configuration has a `group_base` specified. - [This configuration](ldap-ee.md#group-sync) is required for group sync to work properly. -- Ensure the correct [LDAP group link is added to the GitLab - group](ldap-ee.md#adding-group-links). -- Check that the user has an LDAP identity: - 1. Sign in to GitLab as an administrator user. - 1. Navigate to **Admin area -> Users**. - 1. Search for the user - 1. Open the user, by clicking on their name. Do not click 'Edit'. - 1. Navigate to the **Identities** tab. There should be an LDAP identity with - an LDAP DN as the 'Identifier'. If not, this user hasn't logged in with - LDAP yet and must do so first. -- You've waited an hour or [the configured - interval](ldap-ee.md#adjusting-ldap-group-sync-schedule) for the group to - sync. To speed up the process, either go to the GitLab group **Settings -> - Members** and press **Sync now** (sync one group) or [run the group sync Rake - task](../raketasks/ldap.md#run-a-group-sync-starter-only) (sync all groups). - -If all of the above looks good, jump in to a little more advanced debugging in -the rails console. - -1. Enter the [rails console](#rails-console). -1. Choose a GitLab group to test with. This group should have an LDAP group link - already configured. -1. [Enable debug logging, find the above GitLab group, and sync it with LDAP](#sync-one-group-starter-only). -1. Look through the output of the sync. See [example log - output](#example-console-output-after-a-group-sync-starter-only) - for how to read the output. -1. If you still aren't able to see why the user isn't being added, [query the - LDAP group directly](#query-a-group-in-ldap-starter-only) to see what members are listed. -1. Is the user's DN or UID in one of the lists from the above output? One of the DNs or - UIDs here should match the 'Identifier' from the LDAP identity checked earlier. If it doesn't, - the user does not appear to be in the LDAP group. - -#### Admin privileges not granted - -When [Administrator sync](ldap-ee.md#administrator-sync) has been configured -but the configured users aren't granted the correct admin privileges, confirm -the following are true: - -- A [`group_base` is also configured](ldap-ee.md#group-sync). -- The configured `admin_group` in the `gitlab.rb` is a CN, rather than a DN or an array. -- This CN falls under the scope of the configured `group_base`. -- The members of the `admin_group` have already logged into GitLab with their LDAP - credentials. GitLab will only grant this admin access to the users whose - accounts are already connected to LDAP. - -If all the above are true and the users are still not getting access, [run a manual -group sync](#sync-all-groups-starter-only) in the rails console and [look through the -output](#example-console-output-after-a-group-sync-starter-only) to see what happens when -GitLab syncs the `admin_group`. - -#### Sync all groups **(STARTER ONLY)** - -NOTE: **NOTE:** -To sync all groups manually when debugging is unnecessary, [use the Rake -task](../raketasks/ldap.md#run-a-group-sync-starter-only) instead. - -The output from a manual [group sync](ldap-ee.md#group-sync) can show you what happens -when GitLab syncs its LDAP group memberships against LDAP. - -```ruby -Rails.logger.level = Logger::DEBUG - -LdapAllGroupsSyncWorker.new.perform -``` - -Next, [learn how to read the -output](#example-console-output-after-a-group-sync-starter-only). - -##### Example console output after a group sync **(STARTER ONLY)** - -Like the output from the user sync, the output from the [manual group -sync](#sync-all-groups-starter-only) will also be very verbose. However, it contains lots -of helpful information. - -Indicates the point where syncing actually begins: - -```shell -Started syncing 'ldapmain' provider for 'my_group' group -``` - -The following entry shows an array of all user DNs GitLab sees in the LDAP server. -Note that these are the users for a single LDAP group, not a GitLab group. If -you have multiple LDAP groups linked to this GitLab group, you will see multiple -log entries like this - one for each LDAP group. If you don't see an LDAP user -DN in this log entry, LDAP is not returning the user when we do the lookup. -Verify the user is actually in the LDAP group. - -```shell -Members in 'ldap_group_1' LDAP group: ["uid=john0,ou=people,dc=example,dc=com", -"uid=mary0,ou=people,dc=example,dc=com", "uid=john1,ou=people,dc=example,dc=com", -"uid=mary1,ou=people,dc=example,dc=com", "uid=john2,ou=people,dc=example,dc=com", -"uid=mary2,ou=people,dc=example,dc=com", "uid=john3,ou=people,dc=example,dc=com", -"uid=mary3,ou=people,dc=example,dc=com", "uid=john4,ou=people,dc=example,dc=com", -"uid=mary4,ou=people,dc=example,dc=com"] -``` - -Shortly after each of the above entries, you will see a hash of resolved member -access levels. This hash represents all user DNs GitLab thinks should have -access to this group, and at which access level (role). This hash is additive, -and more DNs may be added, or existing entries modified, based on additional -LDAP group lookups. The very last occurrence of this entry should indicate -exactly which users GitLab believes should be added to the group. - -NOTE: **Note:** -10 is 'Guest', 20 is 'Reporter', 30 is 'Developer', 40 is 'Maintainer' -and 50 is 'Owner'. - -```shell -Resolved 'my_group' group member access: {"uid=john0,ou=people,dc=example,dc=com"=>30, -"uid=mary0,ou=people,dc=example,dc=com"=>30, "uid=john1,ou=people,dc=example,dc=com"=>30, -"uid=mary1,ou=people,dc=example,dc=com"=>30, "uid=john2,ou=people,dc=example,dc=com"=>30, -"uid=mary2,ou=people,dc=example,dc=com"=>30, "uid=john3,ou=people,dc=example,dc=com"=>30, -"uid=mary3,ou=people,dc=example,dc=com"=>30, "uid=john4,ou=people,dc=example,dc=com"=>30, -"uid=mary4,ou=people,dc=example,dc=com"=>30} -``` - -It's not uncommon to see warnings like the following. These indicate that GitLab -would have added the user to a group, but the user could not be found in GitLab. -Usually this is not a cause for concern. - -If you think a particular user should already exist in GitLab, but you're seeing -this entry, it could be due to a mismatched DN stored in GitLab. See -[User DN and/or email have changed](#user-dn-orand-email-have-changed) to update the user's LDAP identity. - -```shell -User with DN `uid=john0,ou=people,dc=example,dc=com` should have access -to 'my_group' group but there is no user in GitLab with that -identity. Membership will be updated once the user signs in for -the first time. -``` - -Finally, the following entry says syncing has finished for this group: - -```shell -Finished syncing all providers for 'my_group' group -``` - -Once all the configured group links have been synchronized, GitLab will look -for any Administrators or External users to sync: - -```shell -Syncing admin users for 'ldapmain' provider -``` - -The output will look similar to what happens with a single group, and then -this line will indicate the sync is finished: - -```shell -Finished syncing admin users for 'ldapmain' provider -``` - -If [admin sync](ldap-ee.md#administrator-sync) is not configured, you'll see a message -stating as such: - -```shell -No `admin_group` configured for 'ldapmain' provider. Skipping -``` - -#### Sync one group **(STARTER ONLY)** - -[Syncing all groups](#sync-all-groups-starter-only) can produce a lot of noise in the output, which can be -distracting when you're only interested in troubleshooting the memberships of -a single GitLab group. In that case, here's how you can just sync this group -and see its debug output: - -```ruby -Rails.logger.level = Logger::DEBUG - -# Find the GitLab group. -# If the output is `nil`, the group could not be found. -# If a bunch of group attributes are in the output, your group was found successfully. -group = Group.find_by(name: 'my_gitlab_group') - -# Sync this group against LDAP -EE::Gitlab::Auth::Ldap::Sync::Group.execute_all_providers(group) -``` - -The output will be similar to -[that you'd get from syncing all groups](#example-console-output-after-a-group-sync-starter-only). - -#### Query a group in LDAP **(STARTER ONLY)** - -When you'd like to confirm that GitLab can read a LDAP group and see all its members, -you can run the following: - -```ruby -# Find the adapter and the group itself -adapter = Gitlab::Auth::Ldap::Adapter.new('ldapmain') # If `main` is the LDAP provider -ldap_group = EE::Gitlab::Auth::Ldap::Group.find_by_cn('group_cn_here', adapter) - -# Find the members of the LDAP group -ldap_group.member_dns -ldap_group.member_uids -``` - -### User DN or/and email have changed - -When an LDAP user is created in GitLab, their LDAP DN is stored for later reference. - -If GitLab cannot find a user by their DN, it will fall back -to finding the user by their email. If the lookup is successful, GitLab will -update the stored DN to the new value so both values will now match what's in -LDAP. - -If the email has changed and the DN has not, GitLab will find the user with -the DN and update its own record of the user's email to match the one in LDAP. - -However, if the primary email _and_ the DN change in LDAP, then GitLab will -have no way of identifying the correct LDAP record of the user and, as a -result, the user will be blocked. To rectify this, the user's existing -profile will have to be updated with at least one of the new values (primary -email or DN) so the LDAP record can be found. - -The following script will update the emails for all provided users so they -won't be blocked or unable to access their accounts. - ->**NOTE**: The following script will require that any new accounts with the new -email address are removed first. This is because emails have to be unique in GitLab. - -Go to the [rails console](#rails-console) and then run: - -```ruby -# Each entry will have to include the old username and the new email -emails = { - 'ORIGINAL_USERNAME' => 'NEW_EMAIL_ADDRESS', - ... -} - -emails.each do |username, email| - user = User.find_by_username(username) - user.email = email - user.skip_reconfirmation! - user.save! -end -``` - -You can then [run a UserSync](#sync-all-users-starter-only) **(STARTER ONLY)** to sync the latest DN -for each of these users. - -## Debugging Tools - -### LDAP check - -The [Rake task to check LDAP](../raketasks/ldap.md#check) is a valuable tool -to help determine whether GitLab can successfully establish a connection to -LDAP and can get so far as to even read users. - -If a connection can't be established, it is likely either because of a problem -with your configuration or a firewall blocking the connection. - -- Ensure you don't have a firewall blocking the -connection, and that the LDAP server is accessible to the GitLab host. -- Look for an error message in the Rake check output, which may lead to your LDAP configuration to -confirm that the configuration values (specifically `host`, `port`, `bind_dn`, and -`password`) are correct. -- Look for [errors](#connection) in [the logs](#gitlab-logs) to further debug connection failures. - -If GitLab can successfully connect to LDAP but doesn't return any -users, [see what to do when no users are found](#no-users-are-found). - -### GitLab logs - -If a user account is blocked or unblocked due to the LDAP configuration, a -message will be [logged to `application.log`](../logs.md#applicationlog). - -If there is an unexpected error during an LDAP lookup (configuration error, -timeout), the login is rejected and a message will be [logged to -`production.log`](../logs.md#productionlog). - -### ldapsearch - -`ldapsearch` is a utility that will allow you to query your LDAP server. You can -use it to test your LDAP settings and ensure that the settings you're using -will get you the results you expect. - -When using `ldapsearch`, be sure to use the same settings you've already -specified in your `gitlab.rb` configuration so you can confirm what happens -when those exact settings are used. - -Running this command on the GitLab host will also help confirm that there's no -obstruction between the GitLab host and LDAP. - -For example, consider the following GitLab configuration: - -```shell -gitlab_rails['ldap_servers'] = YAML.load <<-'EOS' # remember to close this block with 'EOS' below - main: # 'main' is the GitLab 'provider ID' of this LDAP server - label: 'LDAP' - host: '127.0.0.1' - port: 389 - uid: 'uid' - encryption: 'plain' - bind_dn: 'cn=admin,dc=ldap-testing,dc=example,dc=com' - password: 'Password1' - active_directory: true - allow_username_or_email_login: false - block_auto_created_users: false - base: 'dc=ldap-testing,dc=example,dc=com' - user_filter: '' - attributes: - username: ['uid', 'userid', 'sAMAccountName'] - email: ['mail', 'email', 'userPrincipalName'] - name: 'cn' - first_name: 'givenName' - last_name: 'sn' - group_base: 'ou=groups,dc=ldap-testing,dc=example,dc=com' - admin_group: 'gitlab_admin' -EOS -``` - -You would run the following `ldapsearch` to find the `bind_dn` user: - -```shell -ldapsearch -D "cn=admin,dc=ldap-testing,dc=example,dc=com" \ - -w Password1 \ - -p 389 \ - -h 127.0.0.1 \ - -b "dc=ldap-testing,dc=example,dc=com" -``` - -Note that the `bind_dn`, `password`, `port`, `host`, and `base` are all -identical to what's configured in the `gitlab.rb`. - -Please see [the official -`ldapsearch` documentation](https://linux.die.net/man/1/ldapsearch) for more. - -### Rails console - -CAUTION: **CAUTION:** -Please note that it is very easy to create, read, modify, and destroy data on the -rails console, so please be sure to run commands exactly as listed. - -The rails console is a valuable tool to help debug LDAP problems. It allows you to -directly interact with the application by running commands and seeing how GitLab -responds to them. - -Please refer to [this guide](../troubleshooting/debug.md#starting-a-rails-console-session) -for instructions on how to use the rails console. - -#### Enable debug output - -This will provide debug output that will be useful to see -what GitLab is doing and with what. This value is not persisted, and will only -be enabled for this session in the rails console. - -To enable debug output in the rails console, [enter the rails -console](#rails-console) and run: - -```ruby -Rails.logger.level = Logger::DEBUG -``` +This document was moved to [another location](ldap/ldap-troubleshooting.md). diff --git a/doc/administration/auth/ldap.md b/doc/administration/auth/ldap.md index 1e91b539db5..f5565628af1 100644 --- a/doc/administration/auth/ldap.md +++ b/doc/administration/auth/ldap.md @@ -1,555 +1,5 @@ --- -type: reference +redirect_to: 'ldap/index.md' --- -<!-- If the change is EE-specific, put it in `ldap-ee.md`, NOT here. --> - -# LDAP - -GitLab integrates with LDAP to support user authentication. - -This integration works with most LDAP-compliant directory servers, including: - -- Microsoft Active Directory -- Apple Open Directory -- Open LDAP -- 389 Server. - -GitLab Enterprise Editions (EE) include enhanced integration, -including group membership syncing as well as multiple LDAP servers support. - -For more details about EE-specific LDAP features, see the -[LDAP Enterprise Edition documentation](ldap-ee.md). - -NOTE: **Note:** -The information on this page is relevant for both GitLab CE and EE. - -## Overview - -[LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) -stands for **Lightweight Directory Access Protocol**, which is a standard -application protocol for accessing and maintaining distributed directory -information services over an Internet Protocol (IP) network. - -## Security - -GitLab assumes that LDAP users: - -- Are not able to change their LDAP `mail`, `email`, or `userPrincipalName` attribute. - An LDAP user who is allowed to change their email on the LDAP server can potentially - [take over any account](#enabling-ldap-sign-in-for-existing-gitlab-users) - on your GitLab server. -- Have unique email addresses, otherwise it is possible for LDAP users with the same - email address to share the same GitLab account. - -We recommend against using LDAP integration if your LDAP users are -allowed to change their 'mail', 'email' or 'userPrincipalName' attribute on -the LDAP server or share email addresses. - -### User deletion - -If a user is deleted from the LDAP server, they will be blocked in GitLab as -well. Users will be immediately blocked from logging in. However, there is an -LDAP check cache time of one hour (see note) which means users that -are already logged in or are using Git over SSH will still be able to access -GitLab for up to one hour. Manually block the user in the GitLab Admin Area to -immediately block all access. - -NOTE: **Note**: -GitLab Enterprise Edition Starter supports a -[configurable sync time](ldap-ee.md#adjusting-ldap-user-sync-schedule), -with a default of one hour. - -## Git password authentication - -LDAP-enabled users can always authenticate with Git using their GitLab username -or email and LDAP password, even if password authentication for Git is disabled -in the application settings. - -## Google Secure LDAP **(CORE ONLY)** - -> Introduced in GitLab 11.9. - -[Google Cloud Identity](https://cloud.google.com/identity/) provides a Secure -LDAP service that can be configured with GitLab for authentication and group sync. -See [Google Secure LDAP](google_secure_ldap.md) for detailed configuration instructions. - -## Configuration - -NOTE: **Note**: -In GitLab Enterprise Edition Starter, you can configure multiple LDAP servers -to connect to one GitLab server. - -For a complete guide on configuring LDAP with: - -- GitLab Community Edition, see - [How to configure LDAP with GitLab CE](how_to_configure_ldap_gitlab_ce/index.md). -- Enterprise Editions, see - [How to configure LDAP with GitLab EE](how_to_configure_ldap_gitlab_ee/index.md). **(STARTER ONLY)** - -To enable LDAP integration you need to add your LDAP server settings in -`/etc/gitlab/gitlab.rb` or `/home/git/gitlab/config/gitlab.yml` for Omnibus -GitLab and installations from source respectively. - -There is a Rake task to check LDAP configuration. After configuring LDAP -using the documentation below, see [LDAP check Rake task](../raketasks/check.md#ldap-check) -for information on the LDAP check Rake task. - -Prior to version 7.4, GitLab used a different syntax for configuring -LDAP integration. The old LDAP integration syntax still works but may be -removed in a future version. If your `gitlab.rb` or `gitlab.yml` file contains -LDAP settings in both the old syntax and the new syntax, only the __old__ -syntax will be used by GitLab. - -The configuration inside `gitlab_rails['ldap_servers']` below is sensitive to -incorrect indentation. Be sure to retain the indentation given in the example. -Copy/paste can sometimes cause problems. - -NOTE: **Note:** -The `encryption` value `ssl` corresponds to 'Simple TLS' in the LDAP -library. `tls` corresponds to StartTLS, not to be confused with regular TLS. -Normally, if you specify `ssl` it will be on port 636, while `tls` (StartTLS) -would be on port 389. `plain` also operates on port 389. - -NOTE: **Note:** -LDAP users must have an email address set, regardless of whether it is used to log in. - -**Omnibus configuration** - -```ruby -gitlab_rails['ldap_enabled'] = true -gitlab_rails['prevent_ldap_sign_in'] = false -gitlab_rails['ldap_servers'] = YAML.load <<-EOS # remember to close this block with 'EOS' below -## -## 'main' is the GitLab 'provider ID' of this LDAP server -## -main: - ## - ## A human-friendly name for your LDAP server. It is OK to change the label later, - ## for instance if you find out it is too large to fit on the web page. - ## - ## Example: 'Paris' or 'Acme, Ltd.' - ## - label: 'LDAP' - - ## - ## Example: 'ldap.mydomain.com' - ## - host: '_your_ldap_server' - - ## - ## This port is an example, it is sometimes different but it is always an - ## integer and not a string. - ## - port: 389 # usually 636 for SSL - uid: 'sAMAccountName' # This should be the attribute, not the value that maps to uid. - - ## - ## Examples: 'america\momo' or 'CN=Gitlab Git,CN=Users,DC=mydomain,DC=com' - ## - bind_dn: '_the_full_dn_of_the_user_you_will_bind_with' - password: '_the_password_of_the_bind_user' - - ## - ## Encryption method. The "method" key is deprecated in favor of - ## "encryption". - ## - ## Examples: "start_tls" or "simple_tls" or "plain" - ## - ## Deprecated values: "tls" was replaced with "start_tls" and "ssl" was - ## replaced with "simple_tls". - ## - ## - encryption: 'plain' - - ## - ## Enables SSL certificate verification if encryption method is - ## "start_tls" or "simple_tls". Defaults to true since GitLab 10.0 for - ## security. This may break installations upon upgrade to 10.0, that did - ## not know their LDAP SSL certificates were not set up properly. - ## - verify_certificates: true - - # OpenSSL::SSL::SSLContext options. - tls_options: - # Specifies the path to a file containing a PEM-format CA certificate, - # e.g. if you need to use an internal CA. - # - # Example: '/etc/ca.pem' - # - ca_file: '' - - # Specifies the SSL version for OpenSSL to use, if the OpenSSL default - # is not appropriate. - # - # Example: 'TLSv1_1' - # - ssl_version: '' - - # Specific SSL ciphers to use in communication with LDAP servers. - # - # Example: 'ALL:!EXPORT:!LOW:!aNULL:!eNULL:!SSLv2' - ciphers: '' - - # Client certificate - # - # Example: - # cert: | - # -----BEGIN CERTIFICATE----- - # MIIDbDCCAlSgAwIBAgIGAWkJxLmKMA0GCSqGSIb3DQEBCwUAMHcxFDASBgNVBAoTC0dvb2dsZSBJ - # bmMuMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UE - # CxMGR1N1aXRlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xOTAyMjAwNzE4 - # rntnF4d+0dd7zP3jrWkbdtoqjLDT/5D7NYRmVCD5vizV98FJ5//PIHbD1gL3a9b2MPAc6k7NV8tl - # ... - # 4SbuJPAiJxC1LQ0t39dR6oMCAMab3hXQqhL56LrR6cRBp6Mtlphv7alu9xb/x51y2x+g2zWtsf80 - # Jrv/vKMsIh/sAyuogb7hqMtp55ecnKxceg== - # -----END CERTIFICATE ----- - cert: '' - - # Client private key - # key: | - # -----BEGIN PRIVATE KEY----- - # MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3DmJtLRmJGY4xU1QtI3yjvxO6 - # bNuyE4z1NF6Xn7VSbcAaQtavWQ6GZi5uukMo+W5DHVtEkgDwh92ySZMuJdJogFbNvJvHAayheCdN - # 7mCQ2UUT9jGXIbmksUn9QMeJVXTZjgJWJzPXToeUdinx9G7+lpVa62UATEd1gaI3oyL72WmpDy/C - # rntnF4d+0dd7zP3jrWkbdtoqjLDT/5D7NYRmVCD5vizV98FJ5//PIHbD1gL3a9b2MPAc6k7NV8tl - # ... - # +9IhSYX+XIg7BZOVDeYqlPfxRvQh8vy3qjt/KUihmEPioAjLaGiihs1Fk5ctLk9A2hIUyP+sEQv9 - # l6RG+a/mW+0rCWn8JAd464Ps9hE= - # -----END PRIVATE KEY----- - key: '' - - ## - ## Set a timeout, in seconds, for LDAP queries. This helps avoid blocking - ## a request if the LDAP server becomes unresponsive. - ## A value of 0 means there is no timeout. - ## - timeout: 10 - - ## - ## This setting specifies if LDAP server is Active Directory LDAP server. - ## For non AD servers it skips the AD specific queries. - ## If your LDAP server is not AD, set this to false. - ## - active_directory: true - - ## - ## If allow_username_or_email_login is enabled, GitLab will ignore everything - ## after the first '@' in the LDAP username submitted by the user on login. - ## - ## Example: - ## - the user enters 'jane.doe@example.com' and 'p@ssw0rd' as LDAP credentials; - ## - GitLab queries the LDAP server with 'jane.doe' and 'p@ssw0rd'. - ## - ## If you are using "uid: 'userPrincipalName'" on ActiveDirectory you need to - ## disable this setting, because the userPrincipalName contains an '@'. - ## - allow_username_or_email_login: false - - ## - ## To maintain tight control over the number of active users on your GitLab installation, - ## enable this setting to keep new users blocked until they have been cleared by the admin - ## (default: false). - ## - block_auto_created_users: false - - ## - ## Base where we can search for users - ## - ## Ex. 'ou=People,dc=gitlab,dc=example' or 'DC=mydomain,DC=com' - ## - ## - base: '' - - ## - ## Filter LDAP users - ## - ## Format: RFC 4515 https://tools.ietf.org/search/rfc4515 - ## Ex. (employeeType=developer) - ## - ## Note: GitLab does not support omniauth-ldap's custom filter syntax. - ## - ## Example for getting only specific users: - ## '(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))' - ## - user_filter: '' - - ## - ## LDAP attributes that GitLab will use to create an account for the LDAP user. - ## The specified attribute can either be the attribute name as a string (e.g. 'mail'), - ## or an array of attribute names to try in order (e.g. ['mail', 'email']). - ## Note that the user's LDAP login will always be the attribute specified as `uid` above. - ## - attributes: - ## - ## The username will be used in paths for the user's own projects - ## (like `gitlab.example.com/username/project`) and when mentioning - ## them in issues, merge request and comments (like `@username`). - ## If the attribute specified for `username` contains an email address, - ## the GitLab username will be the part of the email address before the '@'. - ## - username: ['uid', 'userid', 'sAMAccountName'] - email: ['mail', 'email', 'userPrincipalName'] - - ## - ## If no full name could be found at the attribute specified for `name`, - ## the full name is determined using the attributes specified for - ## `first_name` and `last_name`. - ## - name: 'cn' - first_name: 'givenName' - last_name: 'sn' - - ## - ## If lowercase_usernames is enabled, GitLab will lower case the username. - ## - lowercase_usernames: false - - ## - ## EE only - ## - - ## Base where we can search for groups - ## - ## Ex. ou=groups,dc=gitlab,dc=example - ## - group_base: '' - - ## The CN of a group containing GitLab administrators - ## - ## Ex. administrators - ## - ## Note: Not `cn=administrators` or the full DN - ## - admin_group: '' - - ## An array of CNs of groups containing users that should be considered external - ## - ## Ex. ['interns', 'contractors'] - ## - ## Note: Not `cn=interns` or the full DN - ## - external_groups: [] - - ## - ## The LDAP attribute containing a user's public SSH key - ## - ## Example: sshPublicKey - ## - sync_ssh_keys: false - -## GitLab EE only: add more LDAP servers -## Choose an ID made of a-z and 0-9 . This ID will be stored in the database -## so that GitLab can remember which LDAP server a user belongs to. -#uswest2: -# label: -# host: -# .... -EOS -``` - -**Source configuration** - -Use the same format as `gitlab_rails['ldap_servers']` for the contents under -`servers:` in the example below: - -```yaml -production: - # snip... - ldap: - enabled: false - prevent_ldap_sign_in: false - servers: - ## - ## 'main' is the GitLab 'provider ID' of this LDAP server - ## - main: - ## - ## A human-friendly name for your LDAP server. It is OK to change the label later, - ## for instance if you find out it is too large to fit on the web page. - ## - ## Example: 'Paris' or 'Acme, Ltd.' - label: 'LDAP' - ## snip... -``` - -## Using an LDAP filter to limit access to your GitLab server - -If you want to limit all GitLab access to a subset of the LDAP users on your -LDAP server, the first step should be to narrow the configured `base`. However, -it is sometimes necessary to filter users further. In this case, you can set up -an LDAP user filter. The filter must comply with -[RFC 4515](https://tools.ietf.org/search/rfc4515). - -**Omnibus configuration** - -```ruby -gitlab_rails['ldap_servers'] = YAML.load <<-EOS -main: - # snip... - user_filter: '(employeeType=developer)' -EOS -``` - -**Source configuration** - -```yaml -production: - ldap: - servers: - main: - # snip... - user_filter: '(employeeType=developer)' -``` - -Tip: If you want to limit access to the nested members of an Active Directory -group, you can use the following syntax: - -```plaintext -(memberOf:1.2.840.113556.1.4.1941:=CN=My Group,DC=Example,DC=com) -``` - -Find more information about this "LDAP_MATCHING_RULE_IN_CHAIN" filter at -<https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax>. Support for -nested members in the user filter should not be confused with -[group sync nested groups support](ldap-ee.md#supported-ldap-group-typesattributes). **(STARTER ONLY)** - -Please note that GitLab does not support the custom filter syntax used by -OmniAuth LDAP. - -### Escaping special characters - -The `user_filter` DN can contain special characters. For example: - -- A comma: - - ```plaintext - OU=GitLab, Inc,DC=gitlab,DC=com - ``` - -- Open and close brackets: - - ```plaintext - OU=Gitlab (Inc),DC=gitlab,DC=com - ``` - - These characters must be escaped as documented in - [RFC 4515](https://tools.ietf.org/search/rfc4515). - -- Escape commas with `\2C`. For example: - - ```plaintext - OU=GitLab\2C Inc,DC=gitlab,DC=com - ``` - -- Escape open and close brackets with `\28` and `\29`, respectively. For example: - - ```plaintext - OU=Gitlab \28Inc\29,DC=gitlab,DC=com - ``` - -## Enabling LDAP sign-in for existing GitLab users - -When a user signs in to GitLab with LDAP for the first time, and their LDAP -email address is the primary email address of an existing GitLab user, then -the LDAP DN will be associated with the existing user. If the LDAP email -attribute is not found in GitLab's database, a new user is created. - -In other words, if an existing GitLab user wants to enable LDAP sign-in for -themselves, they should check that their GitLab email address matches their -LDAP email address, and then sign into GitLab via their LDAP credentials. - -## Enabling LDAP username lowercase - -Some LDAP servers, depending on their configurations, can return uppercase usernames. -This can lead to several confusing issues such as creating links or namespaces with uppercase names. - -GitLab can automatically lowercase usernames provided by the LDAP server by enabling -the configuration option `lowercase_usernames`. By default, this configuration option is `false`. - -**Omnibus configuration** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - # snip... - lowercase_usernames: true - EOS - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -**Source configuration** - -1. Edit `config/gitlab.yaml`: - - ```yaml - production: - ldap: - servers: - main: - # snip... - lowercase_usernames: true - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## Disable LDAP web sign in - -It can be useful to prevent using LDAP credentials through the web UI when -an alternative such as SAML is preferred. This allows LDAP to be used for group -sync, while also allowing your SAML identity provider to handle additional -checks like custom 2FA. - -When LDAP web sign in is disabled, users will not see a **LDAP** tab on the sign in page. -This does not disable [using LDAP credentials for Git access](#git-password-authentication). - -**Omnibus configuration** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['prevent_ldap_sign_in'] = true - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -**Source configuration** - -1. Edit `config/gitlab.yaml`: - - ```yaml - production: - ldap: - prevent_ldap_sign_in: true - ``` - -1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -## Encryption - -### TLS Server Authentication - -There are two encryption methods, `simple_tls` and `start_tls`. - -For either encryption method, if setting `verify_certificates: false`, TLS -encryption is established with the LDAP server before any LDAP-protocol data is -exchanged but no validation of the LDAP server's SSL certificate is performed. - ->**Note**: Before GitLab 9.5, `verify_certificates: false` is the default if -unspecified. - -## Limitations - -### TLS Client Authentication - -Not implemented by `Net::LDAP`. -You should disable anonymous LDAP authentication and enable simple or SASL -authentication. The TLS client authentication setting in your LDAP server cannot -be mandatory and clients cannot be authenticated with the TLS protocol. - -## Troubleshooting - -Please see our [administrator guide to troubleshooting LDAP](ldap-troubleshooting.md). +This document was moved to [another location](ldap/index.md). diff --git a/doc/administration/auth/ldap/google_secure_ldap.md b/doc/administration/auth/ldap/google_secure_ldap.md new file mode 100644 index 00000000000..2271ce93b6f --- /dev/null +++ b/doc/administration/auth/ldap/google_secure_ldap.md @@ -0,0 +1,222 @@ +--- +type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Google Secure LDAP **(CORE ONLY)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/46391) in GitLab 11.9. + +[Google Cloud Identity](https://cloud.google.com/identity/) provides a Secure +LDAP service that can be configured with GitLab for authentication and group sync. + +Secure LDAP requires a slightly different configuration than standard LDAP servers. +The steps below cover: + +- Configuring the Secure LDAP Client in the Google Admin console. +- Required GitLab configuration. + +## Configuring Google LDAP client + +1. Navigate to <https://admin.google.com/Dashboard> and sign in as a GSuite domain administrator. + +1. Go to **Apps > LDAP > Add Client**. + +1. Provide an `LDAP client name` and an optional `Description`. Any descriptive + values are acceptable. For example, the name could be 'GitLab' and the + description could be 'GitLab LDAP Client'. Click the **Continue** button. + +  + +1. Set **Access Permission** according to your needs. You must choose either + 'Entire domain (GitLab)' or 'Selected organizational units' for both 'Verify user + credentials' and 'Read user information'. Select 'Add LDAP Client' + + TIP: **Tip:** If you plan to use GitLab [LDAP Group Sync](index.md#group-sync-starter-only) + , turn on 'Read group information'. + +  + +1. Download the generated certificate. This is required for GitLab to + communicate with the Google Secure LDAP service. Save the downloaded certificates + for later use. After downloading, click the **Continue to Client Details** button. + +1. Expand the **Service Status** section and turn the LDAP client 'ON for everyone'. + After selecting 'Save', click on the 'Service Status' bar again to collapse + and return to the rest of the settings. + +1. Expand the **Authentication** section and choose 'Generate New Credentials'. + Copy/note these credentials for later use. After selecting 'Close', click + on the 'Authentication' bar again to collapse and return to the rest of the settings. + +Now the Google Secure LDAP Client configuration is finished. The screenshot below +shows an example of the final settings. Continue on to configure GitLab. + + + +## Configuring GitLab + +Edit GitLab configuration, inserting the access credentials and certificate +obtained earlier. + +The following are the configuration keys that need to be modified using the +values obtained during the LDAP client configuration earlier: + +- `bind_dn`: The access credentials username +- `password`: The access credentials password +- `cert`: The `.crt` file text from the downloaded certificate bundle +- `key`: The `.key` file text from the downloaded certificate bundle + +**For Omnibus installations** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_enabled'] = true + gitlab_rails['ldap_servers'] = YAML.load <<-EOS # remember to close this block with 'EOS' below + main: # 'main' is the GitLab 'provider ID' of this LDAP server + label: 'Google Secure LDAP' + + host: 'ldap.google.com' + port: 636 + uid: 'uid' + bind_dn: 'DizzyHorse' + password: 'd6V5H8nhMUW9AuDP25abXeLd' + encryption: 'simple_tls' + verify_certificates: true + + tls_options: + cert: | + -----BEGIN CERTIFICATE----- + MIIDbDCCAlSgAwIBAgIGAWlzxiIfMA0GCSqGSIb3DQEBCwUAMHcxFDASBgNVBAoTC0dvb2dsZSBJ + bmMuMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UE + CxMGR1N1aXRlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xOTAzMTIyMTE5 + MThaFw0yMjAzMTEyMTE5MThaMHcxFDASBgNVBAoTC0dvb2dsZSBJbmMuMRYwFAYDVQQHEw1Nb3Vu + dGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UECxMGR1N1aXRlMQswCQYDVQQG + EwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB + ALOTy4aC38dyjESk6N8fRsKk8DN23ZX/GaNFL5OUmmA1KWzrvVC881OzNdtGm3vNOIxr9clteEG/ + tQwsmsJvQT5U+GkBt+tGKF/zm7zueHUYqTP7Pg5pxAnAei90qkIRFi17ulObyRHPYv1BbCt8pxNB + 4fG/gAXkFbCNxwh1eiQXXRTfruasCZ4/mHfX7MVm8JmWU9uAVIOLW+DSWOFhrDQduJdGBXJOyC2r + Gqoeg9+tkBmNH/jjxpnEkFW8q7io9DdOUqqNgoidA1h9vpKTs3084sy2DOgUvKN9uXWx14uxIyYU + Y1DnDy0wczcsuRt7l+EgtCEgpsLiLJQbKW+JS1UCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAf60J + yazhbHkDKIH2gFxfm7QLhhnqsmafvl4WP7JqZt0u0KdnvbDPfokdkM87yfbKJU1MTI86M36wEC+1 + P6bzklKz7kXbzAD4GggksAzxsEE64OWHC+Y64Tkxq2NiZTw/76POkcg9StiIXjG0ZcebHub9+Ux/ + rTncip92nDuvgEM7lbPFKRIS/YMhLCk09B/U0F6XLsf1yYjyf5miUTDikPkov23b/YGfpc8kh6hq + 1kqdi6a1cYPP34eAhtRhMqcZU9qezpJF6s9EeN/3YFfKzLODFSsVToBRAdZgGHzj//SAtLyQTD4n + KCSvK1UmaMxNaZyTHg8JnMf0ZuRpv26iSg== + -----END CERTIFICATE----- + + key: | + -----BEGIN PRIVATE KEY----- + MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCzk8uGgt/HcoxEpOjfH0bCpPAz + dt2V/xmjRS+TlJpgNSls671QvPNTszXbRpt7zTiMa/XJbXhBv7UMLJrCb0E+VPhpAbfrRihf85u8 + 7nh1GKkz+z4OacQJwHovdKpCERYte7pTm8kRz2L9QWwrfKcTQeHxv4AF5BWwjccIdXokF10U367m + rAmeP5h31+zFZvCZllPbgFSDi1vg0ljhYaw0HbiXRgVyTsgtqxqqHoPfrZAZjR/448aZxJBVvKu4 + qPQ3TlKqjYKInQNYfb6Sk7N9POLMtgzoFLyjfbl1sdeLsSMmFGNQ5w8tMHM3LLkbe5fhILQhIKbC + 4iyUGylviUtVAgMBAAECggEAIPb0CQy0RJoX+q/lGbRVmnyJpYDf+115WNnl+mrwjdGkeZyqw4v0 + BPzkWYzUFP1esJRO6buBNFybQRFdFW0z5lvVv/zzRKq71aVUBPInxaMRyHuJ8D5lIL8nDtgVOwyE + 7DOGyDtURUMzMjdUwoTe7K+O6QBU4X/1pVPZYgmissYSMmt68LiP8k0p601F4+r5xOi/QEy44aVp + aOJZBUOisKB8BmUXZqmQ4Cy05vU9Xi1rLyzkn9s7fxnZ+JO6Sd1r0Thm1mE0yuPgxkDBh/b4f3/2 + GsQNKKKCiij/6TfkjnBi8ZvWR44LnKpu760g/K7psVNrKwqJG6C/8RAcgISWQQKBgQDop7BaKGhK + 1QMJJ/vnlyYFTucfGLn6bM//pzTys5Gop0tpcfX/Hf6a6Dd+zBhmC3tBmhr80XOX/PiyAIbc0lOI + 31rafZuD/oVx5mlIySWX35EqS14LXmdVs/5vOhsInNgNiE+EPFf1L9YZgG/zA7OUBmqtTeYIPDVC + 7ViJcydItQKBgQDFmK0H0IA6W4opGQo+zQKhefooqZ+RDk9IIZMPOAtnvOM7y3rSVrfsSjzYVuMS + w/RP/vs7rwhaZejnCZ8/7uIqwg4sdUBRzZYR3PRNFeheW+BPZvb+2keRCGzOs7xkbF1mu54qtYTa + HZGZj1OsD83AoMwVLcdLDgO1kw32dkS8IQKBgFRdgoifAHqqVah7VFB9se7Y1tyi5cXWsXI+Wufr + j9U9nQ4GojK52LqpnH4hWnOelDqMvF6TQTyLIk/B+yWWK26Ft/dk9wDdSdystd8L+dLh4k0Y+Whb + +lLMq2YABw+PeJUnqdYE38xsZVHoDjBsVjFGRmbDybeQxauYT7PACy3FAoGBAK2+k9bdNQMbXp7I + j8OszHVkJdz/WXlY1cmdDAxDwXOUGVKIlxTAf7TbiijILZ5gg0Cb+hj+zR9/oI0WXtr+mAv02jWp + W8cSOLS4TnBBpTLjIpdu+BwbnvYeLF6MmEjNKEufCXKQbaLEgTQ/XNlchBSuzwSIXkbWqdhM1+gx + EjtBAoGARAdMIiDMPWIIZg3nNnFebbmtBP0qiBsYohQZ+6i/8s/vautEHBEN6Q0brIU/goo+nTHc + t9VaOkzjCmAJSLPUanuBC8pdYgLu5J20NXUZLD9AE/2bBT3OpezKcdYeI2jqoc1qlWHlNtVtdqQ2 + AcZSFJQjdg5BTyvdEDhaYUKGdRw= + -----END PRIVATE KEY----- + EOS + ``` + +1. Save the file and [reconfigure](../../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. + +--- + +**For installations from source** + +1. Edit `config/gitlab.yml`: + + ```yaml + ldap: + enabled: true + servers: + main: # 'main' is the GitLab 'provider ID' of this LDAP server + label: 'Google Secure LDAP' + + host: 'ldap.google.com' + port: 636 + uid: 'uid' + bind_dn: 'DizzyHorse' + password: 'd6V5H8nhMUW9AuDP25abXeLd' + encryption: 'simple_tls' + verify_certificates: true + + tls_options: + cert: | + -----BEGIN CERTIFICATE----- + MIIDbDCCAlSgAwIBAgIGAWlzxiIfMA0GCSqGSIb3DQEBCwUAMHcxFDASBgNVBAoTC0dvb2dsZSBJ + bmMuMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UE + CxMGR1N1aXRlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xOTAzMTIyMTE5 + MThaFw0yMjAzMTEyMTE5MThaMHcxFDASBgNVBAoTC0dvb2dsZSBJbmMuMRYwFAYDVQQHEw1Nb3Vu + dGFpbiBWaWV3MRQwEgYDVQQDEwtMREFQIENsaWVudDEPMA0GA1UECxMGR1N1aXRlMQswCQYDVQQG + EwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB + ALOTy4aC38dyjESk6N8fRsKk8DN23ZX/GaNFL5OUmmA1KWzrvVC881OzNdtGm3vNOIxr9clteEG/ + tQwsmsJvQT5U+GkBt+tGKF/zm7zueHUYqTP7Pg5pxAnAei90qkIRFi17ulObyRHPYv1BbCt8pxNB + 4fG/gAXkFbCNxwh1eiQXXRTfruasCZ4/mHfX7MVm8JmWU9uAVIOLW+DSWOFhrDQduJdGBXJOyC2r + Gqoeg9+tkBmNH/jjxpnEkFW8q7io9DdOUqqNgoidA1h9vpKTs3084sy2DOgUvKN9uXWx14uxIyYU + Y1DnDy0wczcsuRt7l+EgtCEgpsLiLJQbKW+JS1UCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAf60J + yazhbHkDKIH2gFxfm7QLhhnqsmafvl4WP7JqZt0u0KdnvbDPfokdkM87yfbKJU1MTI86M36wEC+1 + P6bzklKz7kXbzAD4GggksAzxsEE64OWHC+Y64Tkxq2NiZTw/76POkcg9StiIXjG0ZcebHub9+Ux/ + rTncip92nDuvgEM7lbPFKRIS/YMhLCk09B/U0F6XLsf1yYjyf5miUTDikPkov23b/YGfpc8kh6hq + 1kqdi6a1cYPP34eAhtRhMqcZU9qezpJF6s9EeN/3YFfKzLODFSsVToBRAdZgGHzj//SAtLyQTD4n + KCSvK1UmaMxNaZyTHg8JnMf0ZuRpv26iSg== + -----END CERTIFICATE----- + + key: | + -----BEGIN PRIVATE KEY----- + MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCzk8uGgt/HcoxEpOjfH0bCpPAz + dt2V/xmjRS+TlJpgNSls671QvPNTszXbRpt7zTiMa/XJbXhBv7UMLJrCb0E+VPhpAbfrRihf85u8 + 7nh1GKkz+z4OacQJwHovdKpCERYte7pTm8kRz2L9QWwrfKcTQeHxv4AF5BWwjccIdXokF10U367m + rAmeP5h31+zFZvCZllPbgFSDi1vg0ljhYaw0HbiXRgVyTsgtqxqqHoPfrZAZjR/448aZxJBVvKu4 + qPQ3TlKqjYKInQNYfb6Sk7N9POLMtgzoFLyjfbl1sdeLsSMmFGNQ5w8tMHM3LLkbe5fhILQhIKbC + 4iyUGylviUtVAgMBAAECggEAIPb0CQy0RJoX+q/lGbRVmnyJpYDf+115WNnl+mrwjdGkeZyqw4v0 + BPzkWYzUFP1esJRO6buBNFybQRFdFW0z5lvVv/zzRKq71aVUBPInxaMRyHuJ8D5lIL8nDtgVOwyE + 7DOGyDtURUMzMjdUwoTe7K+O6QBU4X/1pVPZYgmissYSMmt68LiP8k0p601F4+r5xOi/QEy44aVp + aOJZBUOisKB8BmUXZqmQ4Cy05vU9Xi1rLyzkn9s7fxnZ+JO6Sd1r0Thm1mE0yuPgxkDBh/b4f3/2 + GsQNKKKCiij/6TfkjnBi8ZvWR44LnKpu760g/K7psVNrKwqJG6C/8RAcgISWQQKBgQDop7BaKGhK + 1QMJJ/vnlyYFTucfGLn6bM//pzTys5Gop0tpcfX/Hf6a6Dd+zBhmC3tBmhr80XOX/PiyAIbc0lOI + 31rafZuD/oVx5mlIySWX35EqS14LXmdVs/5vOhsInNgNiE+EPFf1L9YZgG/zA7OUBmqtTeYIPDVC + 7ViJcydItQKBgQDFmK0H0IA6W4opGQo+zQKhefooqZ+RDk9IIZMPOAtnvOM7y3rSVrfsSjzYVuMS + w/RP/vs7rwhaZejnCZ8/7uIqwg4sdUBRzZYR3PRNFeheW+BPZvb+2keRCGzOs7xkbF1mu54qtYTa + HZGZj1OsD83AoMwVLcdLDgO1kw32dkS8IQKBgFRdgoifAHqqVah7VFB9se7Y1tyi5cXWsXI+Wufr + j9U9nQ4GojK52LqpnH4hWnOelDqMvF6TQTyLIk/B+yWWK26Ft/dk9wDdSdystd8L+dLh4k0Y+Whb + +lLMq2YABw+PeJUnqdYE38xsZVHoDjBsVjFGRmbDybeQxauYT7PACy3FAoGBAK2+k9bdNQMbXp7I + j8OszHVkJdz/WXlY1cmdDAxDwXOUGVKIlxTAf7TbiijILZ5gg0Cb+hj+zR9/oI0WXtr+mAv02jWp + W8cSOLS4TnBBpTLjIpdu+BwbnvYeLF6MmEjNKEufCXKQbaLEgTQ/XNlchBSuzwSIXkbWqdhM1+gx + EjtBAoGARAdMIiDMPWIIZg3nNnFebbmtBP0qiBsYohQZ+6i/8s/vautEHBEN6Q0brIU/goo+nTHc + t9VaOkzjCmAJSLPUanuBC8pdYgLu5J20NXUZLD9AE/2bBT3OpezKcdYeI2jqoc1qlWHlNtVtdqQ2 + AcZSFJQjdg5BTyvdEDhaYUKGdRw= + -----END PRIVATE KEY----- + ``` + +1. Save the file and [restart](../../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. + +<!-- ## Troubleshooting + +Include any troubleshooting steps that you can foresee. If you know beforehand what issues +one might have when setting this up, or when something is changed, or on upgrading, it's +important to describe those, too. Think of things that may go wrong and include them here. +This is important to minimize requests for support, and to avoid doc comments with +questions that you know someone might ask. + +Each scenario can be a third-level heading, e.g. `### Getting error message X`. +If you have none to add when creating a doc, leave this section in place +but commented out to help encourage others to add to it in the future. --> diff --git a/doc/administration/auth/img/google_secure_ldap_add_step_1.png b/doc/administration/auth/ldap/img/google_secure_ldap_add_step_1.png Binary files differindex bee9c602a14..bee9c602a14 100644 --- a/doc/administration/auth/img/google_secure_ldap_add_step_1.png +++ b/doc/administration/auth/ldap/img/google_secure_ldap_add_step_1.png diff --git a/doc/administration/auth/img/google_secure_ldap_add_step_2.png b/doc/administration/auth/ldap/img/google_secure_ldap_add_step_2.png Binary files differindex b127410cb8c..b127410cb8c 100644 --- a/doc/administration/auth/img/google_secure_ldap_add_step_2.png +++ b/doc/administration/auth/ldap/img/google_secure_ldap_add_step_2.png diff --git a/doc/administration/auth/img/google_secure_ldap_client_settings.png b/doc/administration/auth/ldap/img/google_secure_ldap_client_settings.png Binary files differindex 868e6645f56..868e6645f56 100644 --- a/doc/administration/auth/img/google_secure_ldap_client_settings.png +++ b/doc/administration/auth/ldap/img/google_secure_ldap_client_settings.png diff --git a/doc/administration/auth/ldap/img/multi_login.gif b/doc/administration/auth/ldap/img/multi_login.gif Binary files differnew file mode 100644 index 00000000000..5aee6090793 --- /dev/null +++ b/doc/administration/auth/ldap/img/multi_login.gif diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md new file mode 100644 index 00000000000..4a7a972596f --- /dev/null +++ b/doc/administration/auth/ldap/index.md @@ -0,0 +1,756 @@ +--- +type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# General LDAP Setup + +GitLab integrates with LDAP to support user authentication. + +This integration works with most LDAP-compliant directory servers, including: + +- Microsoft Active Directory +- Apple Open Directory +- Open LDAP +- 389 Server + +GitLab Enterprise Editions (EE) include enhanced integration, +including group membership syncing as well as multiple LDAP servers support. + +NOTE: **Note:** +[Microsoft Active Directory Trusts](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2008-R2-and-2008/cc771568(v=ws.10)) are not supported. + +## Overview + +[LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) +stands for **Lightweight Directory Access Protocol**, which is a standard +application protocol for accessing and maintaining distributed directory +information services over an Internet Protocol (IP) network. + +## Security **(CORE ONLY)** + +GitLab assumes that LDAP users: + +- Are not able to change their LDAP `mail`, `email`, or `userPrincipalName` attributes. + An LDAP user who is allowed to change their email on the LDAP server can potentially + [take over any account](#enabling-ldap-sign-in-for-existing-gitlab-users-core-only) + on your GitLab server. +- Have unique email addresses, otherwise it is possible for LDAP users with the same + email address to share the same GitLab account. + +We recommend against using LDAP integration if your LDAP users are +allowed to change their 'mail', 'email' or 'userPrincipalName' attribute on +the LDAP server or share email addresses. + +### User deletion **(CORE ONLY)** + +If a user is deleted from the LDAP server, they will be blocked in GitLab as +well. Users will be immediately blocked from logging in. However, there is an +LDAP check cache time of one hour (see note) which means users that +are already logged in or are using Git over SSH will still be able to access +GitLab for up to one hour. Manually block the user in the GitLab Admin Area to +immediately block all access. + +NOTE: **Note**: +GitLab Enterprise Edition Starter supports a +[configurable sync time](#adjusting-ldap-user-sync-schedule-starter-only). + +## Git password authentication **(CORE ONLY)** + +LDAP-enabled users can always authenticate with Git using their GitLab username +or email and LDAP password, even if password authentication for Git is disabled +in the application settings. + +## Enabling LDAP sign-in for existing GitLab users **(CORE ONLY)** + +When a user signs in to GitLab with LDAP for the first time, and their LDAP +email address is the primary email address of an existing GitLab user, then +the LDAP DN will be associated with the existing user. If the LDAP email +attribute is not found in GitLab's database, a new user is created. + +In other words, if an existing GitLab user wants to enable LDAP sign-in for +themselves, they should check that their GitLab email address matches their +LDAP email address, and then sign into GitLab via their LDAP credentials. + +## Google Secure LDAP **(CORE ONLY)** + +> Introduced in GitLab 11.9. + +[Google Cloud Identity](https://cloud.google.com/identity/) provides a Secure +LDAP service that can be configured with GitLab for authentication and group sync. +See [Google Secure LDAP](google_secure_ldap.md) for detailed configuration instructions. + +## Configuration **(CORE ONLY)** + +To enable LDAP integration you need to add your LDAP server settings in +`/etc/gitlab/gitlab.rb` or `/home/git/gitlab/config/gitlab.yml` for Omnibus +GitLab and installations from source respectively. + +There is a Rake task to check LDAP configuration. After configuring LDAP +using the documentation below, see [LDAP check Rake task](../../raketasks/check.md#ldap-check) +for information on the LDAP check Rake task. + +NOTE: **Note:** +The `encryption` value `simple_tls` corresponds to 'Simple TLS' in the LDAP +library. `start_tls` corresponds to StartTLS, not to be confused with regular TLS. +Normally, if you specify `simple_tls` it will be on port 636, while `start_tls` (StartTLS) +would be on port 389. `plain` also operates on port 389. Removed values: `tls` was replaced with `start_tls` and `ssl` was replaced with `simple_tls`. + +NOTE: **Note:** +LDAP users must have an email address set, regardless of whether it is used to log in. + +### Example Configurations **(CORE ONLY)** + +**Omnibus Configuration** + +```ruby +gitlab_rails['ldap_enabled'] = true +gitlab_rails['prevent_ldap_sign_in'] = false +gitlab_rails['ldap_servers'] = { +'main' => { + 'label' => 'LDAP', + 'host' => 'ldap.mydomain.com', + 'port' => 389, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'verify_certificates' => true, + 'bind_dn' => '_the_full_dn_of_the_user_you_will_bind_with', + 'password' => '_the_password_of_the_bind_user', + 'encryption' => 'plain', + 'verify_certificates' => true, + 'tls_options' => { + 'ca_file' => '', + 'ssl_version' => '', + 'ciphers' => '', + 'cert' => '', + 'key' => '' + }, + 'timeout' => 10, + 'active_directory' => true, + 'allow_username_or_email_login' => false, + 'block_auto_created_users' => false, + 'base' => 'dc=example,dc=com', + 'user_filter' => '', + 'attributes' => { + 'username' => ['uid', 'userid', 'sAMAccountName'], + 'email' => ['mail', 'email', 'userPrincipalName'], + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + }, + 'lowercase_usernames' => false, + + # EE Only + 'group_base' => '', + 'admin_group' => '', + 'external_groups' => [], + 'sync_ssh_keys' => false + } +} +``` + +**Source Configuration** + +```yaml +production: + # snip... + ldap: + enabled: false + prevent_ldap_sign_in: false + servers: + main: + label: 'LDAP' + ... +``` + +### Basic Configuration Settings **(CORE ONLY)** + +| Setting | Description | Required | Examples | +| ------- | ----------- | -------- | -------- | +| `label` | A human-friendly name for your LDAP server. It will be displayed on your login page. | yes | `'Paris'` or `'Acme, Ltd.'` | +| `host` | IP address or domain name of your LDAP server. | yes | `'ldap.mydomain.com'` | +| `port` | The port to connect with on your LDAP server. Always an integer, not a string. | yes | `389` or `636` (for SSL) | +| `uid` | LDAP attribute for username. Should be the attribute, not the value that maps to the `uid`. | yes | `'sAMAccountName'`, `'uid'`, `'userPrincipalName'` | +| `bind_dn` | The full DN of the user you will bind with. | no | `'america\momo'` or `'CN=Gitlab,OU=Users,DC=domain,DC=com'` | +| `password` | The password of the bind user. | no | `'your_great_password'` | +| `encryption` | Encryption method. The `method` key is deprecated in favor of `encryption`. | yes | `'start_tls'` or `'simple_tls'` or `'plain'` | +| `verify_certificates` | Enables SSL certificate verification if encryption method is `start_tls` or `simple_tls`. Defaults to true. | no | boolean | +| `timeout` | Set a timeout, in seconds, for LDAP queries. This helps avoid blocking a request if the LDAP server becomes unresponsive. A value of 0 means there is no timeout. | no | `10` or `30` | +| `active_directory` | This setting specifies if LDAP server is Active Directory LDAP server. For non-AD servers it skips the AD specific queries. If your LDAP server is not AD, set this to false. | no | boolean | +| `allow_username_or_email_login` | If enabled, GitLab will ignore everything after the first `@` in the LDAP username submitted by the user on login. If you are using `uid: 'userPrincipalName'` on ActiveDirectory you need to disable this setting, because the userPrincipalName contains an `@`. | no | boolean | +| `block_auto_created_users` | To maintain tight control over the number of active users on your GitLab installation, enable this setting to keep new users blocked until they have been cleared by the admin (default: false). | no | boolean | +| `base` | Base where we can search for users. | yes | `'ou=people,dc=gitlab,dc=example'` or `'DC=mydomain,DC=com'` | +| `user_filter` | Filter LDAP users. Format: [RFC 4515](https://tools.ietf.org/search/rfc4515) Note: GitLab does not support `omniauth-ldap`'s custom filter syntax. | no | `'(employeeType=developer)'` or `'(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))'` | +| `lowercase_usernames` | If lowercase_usernames is enabled, GitLab will lower case the username. | no | boolean | + +### SSL Configuration Settings **(CORE ONLY)** + +| Setting | Description | Required | Examples | +| ------- | ----------- | -------- | -------- | +| `ca_file` | Specifies the path to a file containing a PEM-format CA certificate, e.g. if you need to use an internal CA. | no | `'/etc/ca.pem'` | +| `ssl_version` | Specifies the SSL version for OpenSSL to use, if the OpenSSL default is not appropriate. | no | `'TLSv1_1'` | +| `ciphers` | Specific SSL ciphers to use in communication with LDAP servers. | no | `'ALL:!EXPORT:!LOW:!aNULL:!eNULL:!SSLv2'` | +| `cert` | Client certificate | no | `'-----BEGIN CERTIFICATE----- <REDACTED> -----END CERTIFICATE -----'` | +| `key` | Client private key | no | `'-----BEGIN PRIVATE KEY----- <REDACTED> -----END PRIVATE KEY -----'` | + +### Attribute Configuration Settings **(CORE ONLY)** + +LDAP attributes that GitLab will use to create an account for the LDAP user. The specified attribute can either be the attribute name as a string (e.g. `'mail'`), or an array of attribute names to try in order (e.g. `['mail', 'email']`). Note that the user's LDAP login will always be the attribute specified as `uid` above. + +| Setting | Description | Required | Examples | +| ------- | ----------- | -------- | -------- | +| `username` | The username will be used in paths for the user's own projects (like `gitlab.example.com/username/project`) and when mentioning them in issues, merge request and comments (like `@username`). If the attribute specified for `username` contains an email address, the GitLab username will be the part of the email address before the `@`. | no | `['uid', 'userid', 'sAMAccountName']` | +| `email` | LDAP attribute for user email. | no | `['mail', 'email', 'userPrincipalName']` | +| `name` | LDAP attribute for user display name. If no full name could be found at the attribute specified for `name`, the full name is determined using the attributes specified for `first_name` and `last_name`. | no | `'cn'` or `'displayName'` | +| `first_name` | LDAP attribute for user first name. | no | `'givenName'` | +| `last_name` | LDAP attribute for user last name. | no | `'sn'` | + +### LDAP Sync Configuration Settings **(STARTER ONLY)** + +| Setting | Description | Required | Examples | +| ------- | ----------- | -------- | -------- | +| `group_base` | Base used to search for groups. | no | `'ou=groups,dc=gitlab,dc=example'` | +| `admin_group` | The CN of a group containing GitLab administrators. Note: Not `cn=administrators` or the full DN. | no | `'administrators'` | +| `external_groups` | An array of CNs of groups containing users that should be considered external. Note: Not `cn=interns` or the full DN. | no | `['interns', 'contractors']` | +| `sync_ssh_keys` | The LDAP attribute containing a user's public SSH key. | no | `'sshPublicKey'` or false if not set | + +### Set up LDAP user filter **(CORE ONLY)** + +If you want to limit all GitLab access to a subset of the LDAP users on your +LDAP server, the first step should be to narrow the configured `base`. However, +it is sometimes necessary to filter users further. In this case, you can set up +an LDAP user filter. The filter must comply with +[RFC 4515](https://tools.ietf.org/search/rfc4515). + +**Omnibus configuration** + +```ruby +gitlab_rails['ldap_servers'] = { +'main' => { + # snip... + 'user_filter' => '(employeeType=developer)' + } +} +``` + +**Source configuration** + +```yaml +production: + ldap: + servers: + main: + # snip... + user_filter: '(employeeType=developer)' +``` + +If you want to limit access to the nested members of an Active Directory +group, you can use the following syntax: + +```plaintext +(memberOf:1.2.840.113556.1.4.1941:=CN=My Group,DC=Example,DC=com) +``` + +For more information about this "LDAP_MATCHING_RULE_IN_CHAIN" filter, see the following +[Microsoft Search Filter Syntax](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax) document. +Support for nested members in the user filter should not be confused with +[group sync nested groups support](#supported-ldap-group-typesattributes). **(STARTER ONLY)** + +Please note that GitLab does not support the custom filter syntax used by +OmniAuth LDAP. + +#### Escaping special characters **(CORE ONLY)** + +The `user_filter` DN can contain special characters. For example: + +- A comma: + + ```plaintext + OU=GitLab, Inc,DC=gitlab,DC=com + ``` + +- Open and close brackets: + + ```plaintext + OU=Gitlab (Inc),DC=gitlab,DC=com + ``` + + These characters must be escaped as documented in + [RFC 4515](https://tools.ietf.org/search/rfc4515). + +- Escape commas with `\2C`. For example: + + ```plaintext + OU=GitLab\2C Inc,DC=gitlab,DC=com + ``` + +- Escape open and close brackets with `\28` and `\29`, respectively. For example: + + ```plaintext + OU=Gitlab \28Inc\29,DC=gitlab,DC=com + ``` + +### Enabling LDAP username lowercase **(CORE ONLY)** + +Some LDAP servers, depending on their configurations, can return uppercase usernames. +This can lead to several confusing issues such as creating links or namespaces with uppercase names. + +GitLab can automatically lowercase usernames provided by the LDAP server by enabling +the configuration option `lowercase_usernames`. By default, this configuration option is `false`. + +**Omnibus configuration** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_servers'] = { + 'main' => { + # snip... + 'lowercase_usernames' => true + } + } + ``` + +1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +**Source configuration** + +1. Edit `config/gitlab.yaml`: + + ```yaml + production: + ldap: + servers: + main: + # snip... + lowercase_usernames: true + ``` + +1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + +### Disable LDAP web sign in **(CORE ONLY)** + +It can be useful to prevent using LDAP credentials through the web UI when +an alternative such as SAML is preferred. This allows LDAP to be used for group +sync, while also allowing your SAML identity provider to handle additional +checks like custom 2FA. + +When LDAP web sign in is disabled, users will not see a **LDAP** tab on the sign in page. +This does not disable [using LDAP credentials for Git access](#git-password-authentication-core-only). + +**Omnibus configuration** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['prevent_ldap_sign_in'] = true + ``` + +1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +**Source configuration** + +1. Edit `config/gitlab.yaml`: + + ```yaml + production: + ldap: + prevent_ldap_sign_in: true + ``` + +1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + +## Encryption **(CORE ONLY)** + +### TLS Server Authentication + +There are two encryption methods, `simple_tls` and `start_tls`. + +For either encryption method, if setting `verify_certificates: false`, TLS +encryption is established with the LDAP server before any LDAP-protocol data is +exchanged but no validation of the LDAP server's SSL certificate is performed. + +### Limitations + +#### TLS Client Authentication + +Not implemented by `Net::LDAP`. + +You should disable anonymous LDAP authentication and enable simple or SASL +authentication. The TLS client authentication setting in your LDAP server cannot +be mandatory and clients cannot be authenticated with the TLS protocol. + +## Multiple LDAP servers **(STARTER ONLY)** + +With GitLab Enterprise Edition Starter, you can configure multiple LDAP servers +that your GitLab instance will connect to. + +To add another LDAP server: + +1. Duplicate the settings under [the main configuration](#configuration-core-only). +1. Edit them to match the additional LDAP server. + +Be sure to choose a different provider ID made of letters a-z and numbers 0-9. +This ID will be stored in the database so that GitLab can remember which LDAP +server a user belongs to. + + + +Based on the example illustrated on the image above, +our `gitlab.rb` configuration would look like: + +```ruby +gitlab_rails['ldap_enabled'] = true +gitlab_rails['ldap_servers'] = { +'main' => { + 'label' => 'GitLab AD', + 'host' => 'ad.example.org', + 'port' => 636, + ... + }, + +'secondary' => { + 'label' => 'GitLab Secondary AD', + 'host' => 'ad-secondary.example.net', + 'port' => 636, + ... + }, + +'tertiary' => { + 'label' => 'GitLab Tertiary AD', + 'host' => 'ad-tertiary.example.net', + 'port' => 636, + ... + } + +} +``` + +NOTE: **Note:** +Any number of LDAP servers can be configured. However, make sure to use a unique naming convention for the `label` section of each entry as this will be the display name of the tab shown on the sign-in page. + +## User sync **(STARTER ONLY)** + +Once per day, GitLab runs a worker to check and update GitLab +users against LDAP. + +The process executes the following access checks: + +- Ensure the user is still present in LDAP. +- If the LDAP server is Active Directory, ensure the user is active (not + blocked/disabled state). This will only be checked if + `active_directory: true` is set in the LDAP configuration. + +NOTE: **Note:** +In Active Directory, a user is marked as disabled/blocked if the user +account control attribute (`userAccountControl:1.2.840.113556.1.4.803`) +has bit 2 set. See <https://ctovswild.com/2009/09/03/bitmask-searches-in-ldap/> +for more information. + +The user will be set to `ldap_blocked` state in GitLab if the above conditions +fail. This means the user will not be able to login or push/pull code. + +The process will also update the following user information: + +- Email address. +- If `sync_ssh_keys` is set, SSH public keys. +- If Kerberos is enabled, Kerberos identity. + +NOTE: **Note:** +The LDAP sync process updates existing users while new users are created on first sign in. + +### Adjusting LDAP user sync schedule **(STARTER ONLY)** + +NOTE: **Note:** +These are cron formatted values. You can use a crontab generator to create +these values, for example <http://www.crontabgenerator.com/>. + +By default, GitLab will run a worker once per day at 01:30 a.m. server time to +check and update GitLab users against LDAP. + +You can manually configure LDAP user sync times by setting the +following configuration values. The example below shows how to set LDAP user +sync to run once every 12 hours at the top of the hour. + +**Omnibus installations** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" + ``` + +1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +**Source installations** + +1. Edit `config/gitlab.yaml`: + + ```yaml + cron_jobs: + ldap_sync_worker_cron: + "0 */12 * * *" + ``` + +1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + +## Group Sync **(STARTER ONLY)** + +If your LDAP supports the `memberof` property, when the user signs in for the +first time GitLab will trigger a sync for groups the user should be a member of. +That way they don't need to wait for the hourly sync to be granted +access to their groups and projects. + +A group sync process will run every hour on the hour, and `group_base` must be set +in LDAP configuration for LDAP synchronizations based on group CN to work. This allows +GitLab group membership to be automatically updated based on LDAP group members. + +The `group_base` configuration should be a base LDAP 'container', such as an +'organization' or 'organizational unit', that contains LDAP groups that should +be available to GitLab. For example, `group_base` could be +`ou=groups,dc=example,dc=com`. In the config file it will look like the +following. + +**Omnibus configuration** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_servers'] = { + 'main' => { + # snip... + 'group_base' => 'ou=groups,dc=example,dc=com', + } + } + ``` + +1. [Apply your changes to GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure). + +**Source configuration** + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: + ldap: + servers: + main: + # snip... + group_base: ou=groups,dc=example,dc=com + ``` + +1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + +To take advantage of group sync, group owners or maintainers will need to [create one +or more LDAP group links](#adding-group-links-starter-only). + +### Adding group links **(STARTER ONLY)** + +For information on adding group links via CNs and filters, refer to [the GitLab groups documentation](../../../user/group/index.md#manage-group-memberships-via-ldap). + +### Administrator sync **(STARTER ONLY)** + +As an extension of group sync, you can automatically manage your global GitLab +administrators. Specify a group CN for `admin_group` and all members of the +LDAP group will be given administrator privileges. The configuration will look +like the following. + +NOTE: **Note:** +Administrators will not be synced unless `group_base` is also +specified alongside `admin_group`. Also, only specify the CN of the admin +group, as opposed to the full DN. + +**Omnibus configuration** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_servers'] = { + 'main' => { + # snip... + 'group_base' => 'ou=groups,dc=example,dc=com', + 'admin_group' => 'my_admin_group', + } + } + ``` + +1. [Apply your changes to GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure). + +**Source configuration** + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: + ldap: + servers: + main: + # snip... + group_base: ou=groups,dc=example,dc=com + admin_group: my_admin_group + ``` + +1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + +### Global group memberships lock **(STARTER ONLY)** + +"Lock memberships to LDAP synchronization" setting allows instance administrators +to lock down user abilities to invite new members to a group. + +When enabled, the following applies: + +- Only administrator can manage memberships of any group including access levels. +- Users are not allowed to share project with other groups or invite members to + a project created in a group. + +### Adjusting LDAP group sync schedule **(STARTER ONLY)** + +NOTE: **Note:** +These are cron formatted values. You can use a crontab generator to create +these values, for example [Crontab Generator](http://www.crontabgenerator.com/). + +By default, GitLab runs a group sync process every hour, on the hour. + +CAUTION: **Important:** +It's recommended that you do not start the sync process too frequently as this +could lead to multiple syncs running concurrently. This is primarily a concern +for installations with a large number of LDAP users. Please review the +[LDAP group sync benchmark metrics](#benchmarks) to see how +your installation compares before proceeding. + +You can manually configure LDAP group sync times by setting the +following configuration values. The example below shows how to set group +sync to run once every 2 hours at the top of the hour. + +**Omnibus installations** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" + ``` + +1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +**Source installations** + +1. Edit `config/gitlab.yaml`: + + ```yaml + cron_jobs: + ldap_group_sync_worker_cron: + "*/30 * * * *" + ``` + +1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + +### External groups **(STARTER ONLY)** + +Using the `external_groups` setting will allow you to mark all users belonging +to these groups as [external users](../../../user/permissions.md#external-users-core-only). +Group membership is checked periodically through the `LdapGroupSync` background +task. + +**Omnibus configuration** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_servers'] = { + 'main' => { + # snip... + 'external_groups' => ['interns', 'contractors'], + } + } + ``` + +1. [Apply your changes to GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure). + +**Source configuration** + +1. Edit `config/gitlab.yaml`: + + ```yaml + production: + ldap: + servers: + main: + # snip... + external_groups: ['interns', 'contractors'] + ``` + +1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + +### Group sync technical details + +There is a lot going on with group sync 'under the hood'. This section +outlines what LDAP queries are executed and what behavior you can expect +from group sync. + +Group member access will be downgraded from a higher level if their LDAP group +membership changes. For example, if a user has 'Owner' rights in a group and the +next group sync reveals they should only have 'Developer' privileges, their +access will be adjusted accordingly. The only exception is if the user is the +*last* owner in a group. Groups need at least one owner to fulfill +administrative duties. + +#### Supported LDAP group types/attributes + +GitLab supports LDAP groups that use member attributes: + +- `member` +- `submember` +- `uniquemember` +- `memberof` +- `memberuid`. + +This means group sync supports, at least, LDAP groups with the following object classes: +`groupOfNames`, `posixGroup`, and `groupOfUniqueNames`. + +Other object classes should work fine as long as members +are defined as one of the mentioned attributes. This also means GitLab supports +Microsoft Active Directory, Apple Open Directory, Open LDAP, and 389 Server. +Other LDAP servers should work, too. + +Active Directory also supports nested groups. Group sync will recursively +resolve membership if `active_directory: true` is set in the configuration file. + +NOTE: **Note:** +Nested group memberships are resolved only if the nested group +is found within the configured `group_base`. For example, if GitLab sees a +nested group with DN `cn=nested_group,ou=special_groups,dc=example,dc=com` but +the configured `group_base` is `ou=groups,dc=example,dc=com`, `cn=nested_group` +is ignored. + +#### Queries + +- Each LDAP group is queried a maximum of one time with base `group_base` and + filter `(cn=<cn_from_group_link>)`. +- If the LDAP group has the `memberuid` attribute, GitLab will execute another + LDAP query per member to obtain each user's full DN. These queries are + executed with base `base`, scope 'base object', and a filter depending on + whether `user_filter` is set. Filter may be `(uid=<uid_from_group>)` or a + joining of `user_filter`. + +#### Benchmarks + +Group sync was written to be as performant as possible. Data is cached, database +queries are optimized, and LDAP queries are minimized. The last benchmark run +revealed the following metrics: + +For 20000 LDAP users, 11000 LDAP groups and 1000 GitLab groups with 10 +LDAP group links each: + +- Initial sync (no existing members assigned in GitLab) took 1.8 hours +- Subsequent syncs (checking membership, no writes) took 15 minutes + +These metrics are meant to provide a baseline and performance may vary based on +any number of factors. This was a pretty extreme benchmark and most instances will +not have near this many users or groups. Disk speed, database performance, +network and LDAP server response time will affect these metrics. + +## Troubleshooting + +Please see our [administrator guide to troubleshooting LDAP](ldap-troubleshooting.md). diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md new file mode 100644 index 00000000000..909802b5dec --- /dev/null +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -0,0 +1,678 @@ +--- +type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# LDAP Troubleshooting for Administrators + +## Common Problems & Workflows + +### Connection + +#### Connection refused + +If you are getting `Connection Refused` errors when trying to connect to the +LDAP server please double-check the LDAP `port` and `encryption` settings used by +GitLab. Common combinations are `encryption: 'plain'` and `port: 389`, OR +`encryption: 'simple_tls'` and `port: 636`. + +#### Connection times out + +If GitLab cannot reach your LDAP endpoint, you will see a message like this: + +```plaintext +Could not authenticate you from Ldapmain because "Connection timed out - user specified timeout". +``` + +If your configured LDAP provider and/or endpoint is offline or otherwise +unreachable by GitLab, no LDAP user will be able to authenticate and log in. +GitLab does not cache or store credentials for LDAP users to provide authentication +during an LDAP outage. + +Contact your LDAP provider or administrator if you are seeing this error. + +#### Referral error + +If you see `LDAP search error: Referral` in the logs, or when troubleshooting +LDAP Group Sync, this error may indicate a configuration problem. The LDAP +configuration `/etc/gitlab/gitlab.rb` (Omnibus) or `config/gitlab.yml` (source) +is in YAML format and is sensitive to indentation. Check that `group_base` and +`admin_group` configuration keys are indented 2 spaces past the server +identifier. The default identifier is `main` and an example snippet looks like +the following: + +```yaml +main: # 'main' is the GitLab 'provider ID' of this LDAP server + label: 'LDAP' + host: 'ldap.example.com' + ... + group_base: 'cn=my_group,ou=groups,dc=example,dc=com' + admin_group: 'my_admin_group' +``` + +#### Query LDAP **(STARTER ONLY)** + +The following allows you to perform a search in LDAP using the rails console. +Depending on what you're trying to do, it may make more sense to query [a +user](#query-a-user-in-ldap) or [a group](#query-a-group-in-ldap-starter-only) directly, or +even [use `ldapsearch`](#ldapsearch) instead. + +```ruby +adapter = Gitlab::Auth::Ldap::Adapter.new('ldapmain') +options = { + # :base is required + # use .base or .group_base + base: adapter.config.group_base, + + # :filter is optional + # 'cn' looks for all "cn"s under :base + # '*' is the search string - here, it's a wildcard + filter: Net::Ldap::Filter.eq('cn', '*'), + + # :attributes is optional + # the attributes we want to get returned + attributes: %w(dn cn memberuid member submember uniquemember memberof) +} +adapter.ldap_search(options) +``` + +For examples of how this is run, +[review the `Adapter` module](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/ee/gitlab/auth/ldap/adapter.rb). + +### User logins + +#### No users are found + +If [you've confirmed](#ldap-check) that a connection to LDAP can be +established but GitLab doesn't show you LDAP users in the output, one of the +following is most likely true: + +- The `bind_dn` user doesn't have enough permissions to traverse the user tree. +- The user(s) don't fall under the [configured `base`](index.md#configuration-core-only). +- The [configured `user_filter`](index.md#set-up-ldap-user-filter-core-only) blocks access to the user(s). + +In this case, you con confirm which of the above is true using +[ldapsearch](#ldapsearch) with the existing LDAP configuration in your +`/etc/gitlab/gitlab.rb`. + +#### User(s) cannot login + +A user can have trouble logging in for any number of reasons. To get started, +here are some questions to ask yourself: + +- Does the user fall under the [configured `base`](index.md#configuration-core-only) in + LDAP? The user must fall under this `base` to login. +- Does the user pass through the [configured `user_filter`](index.md#set-up-ldap-user-filter-core-only)? + If one is not configured, this question can be ignored. If it is, then the + user must also pass through this filter to be allowed to login. + - Refer to our docs on [debugging the `user_filter`](#debug-ldap-user-filter). + +If the above are both okay, the next place to look for the problem is +the logs themselves while reproducing the issue. + +- Ask the user to login and let it fail. +- [Look through the output](#gitlab-logs) for any errors or other + messages about the login. You may see one of the other error messages on + this page, in which case that section can help resolve the issue. + +If the logs don't lead to the root of the problem, use the +[rails console](#rails-console) to [query this user](#query-a-user-in-ldap) +to see if GitLab can read this user on the LDAP server. + +It can also be helpful to +[debug a user sync](#sync-all-users-starter-only) to +investigate further. + +#### Invalid credentials on login + +If that the login credentials used are accurate on LDAP, ensure the following +are true for the user in question: + +- Make sure the user you are binding with has enough permissions to read the user's + tree and traverse it. +- Check that the `user_filter` is not blocking otherwise valid users. +- Run [an LDAP check command](#ldap-check) to make sure that the LDAP settings + are correct and [GitLab can see your users](#no-users-are-found). + +#### Email has already been taken + +A user tries to login with the correct LDAP credentials, is denied access, +and the [production.log](../../logs.md#productionlog) shows an error that looks like this: + +```plaintext +(LDAP) Error saving user <USER DN> (email@example.com): ["Email has already been taken"] +``` + +This error is referring to the email address in LDAP, `email@example.com`. Email +addresses must be unique in GitLab and LDAP links to a user's primary email (as opposed +to any of their possibly-numerous secondary emails). Another user (or even the +same user) has the email `email@example.com` set as a secondary email, which +is throwing this error. + +We can check where this conflicting email address is coming from using the +[rails console](#rails-console). Once in the console, run the following: + +```ruby +# This searches for an email among the primary AND secondary emails +user = User.find_by_any_email('email@example.com') +user.username +``` + +This will show you which user has this email address. One of two steps will +have to be taken here: + +- To create a new GitLab user/username for this user when logging in with LDAP, + remove the secondary email to remove the conflict. +- To use an existing GitLab user/username for this user to use with LDAP, + remove this email as a secondary email and make it a primary one so GitLab + will associate this profile to the LDAP identity. + +The user can do either of these steps [in their +profile](../../../user/profile/index.md#user-profile) or an admin can do it. + +#### Debug LDAP user filter + +[`ldapsearch`](#ldapsearch) allows you to test your configured +[user filter](index.md#set-up-ldap-user-filter-core-only) +to confirm that it returns the users you expect it to return. + +```shell +ldapsearch -H ldaps://$host:$port -D "$bind_dn" -y bind_dn_password.txt -b "$base" "$user_filter" sAMAccountName +``` + +- Variables beginning with a `$` refer to a variable from the LDAP section of + your configuration file. +- Replace `ldaps://` with `ldap://` if you are using the plain authentication method. + Port `389` is the default `ldap://` port and `636` is the default `ldaps://` + port. +- We are assuming the password for the `bind_dn` user is in `bind_dn_password.txt`. + +#### Sync all users **(STARTER ONLY)** + +The output from a manual [user sync](index.md#user-sync-starter-only) can show you what happens when +GitLab tries to sync its users against LDAP. Enter the [rails console](#rails-console) +and then run: + +```ruby +Rails.logger.level = Logger::DEBUG + +LdapSyncWorker.new.perform +``` + +Next, [learn how to read the +output](#example-console-output-after-a-user-sync-starter-only). + +##### Example console output after a user sync **(STARTER ONLY)** + +The output from a [manual user sync](#sync-all-users-starter-only) will be very verbose, and a +single user's successful sync can look like this: + +```shell +Syncing user John, email@example.com + Identity Load (0.9ms) SELECT "identities".* FROM "identities" WHERE "identities"."user_id" = 20 AND (provider LIKE 'ldap%') LIMIT 1 +Instantiating Gitlab::Auth::Ldap::Person with LDIF: +dn: cn=John Smith,ou=people,dc=example,dc=com +cn: John Smith +mail: email@example.com +memberof: cn=admin_staff,ou=people,dc=example,dc=com +uid: John + + UserSyncedAttributesMetadata Load (0.9ms) SELECT "user_synced_attributes_metadata".* FROM "user_synced_attributes_metadata" WHERE "user_synced_attributes_metadata"."user_id" = 20 LIMIT 1 + (0.3ms) BEGIN + Namespace Load (1.0ms) SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."owner_id" = 20 AND "namespaces"."type" IS NULL LIMIT 1 + Route Load (0.8ms) SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 27 AND "routes"."source_type" = 'Namespace' LIMIT 1 + Ci::Runner Load (1.1ms) SELECT "ci_runners".* FROM "ci_runners" INNER JOIN "ci_runner_namespaces" ON "ci_runners"."id" = "ci_runner_namespaces"."runner_id" WHERE "ci_runner_namespaces"."namespace_id" = 27 + (0.7ms) COMMIT + (0.4ms) BEGIN + Route Load (0.8ms) SELECT "routes".* FROM "routes" WHERE (LOWER("routes"."path") = LOWER('John')) + Namespace Load (1.0ms) SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" = 27 LIMIT 1 + Route Exists (0.9ms) SELECT 1 AS one FROM "routes" WHERE LOWER("routes"."path") = LOWER('John') AND "routes"."id" != 50 LIMIT 1 + User Update (1.1ms) UPDATE "users" SET "updated_at" = '2019-10-17 14:40:59.751685', "last_credential_check_at" = '2019-10-17 14:40:59.738714' WHERE "users"."id" = 20 +``` + +There's a lot here, so let's go over what could be helpful when debugging. + +First, GitLab will look for all users that have previously +logged in with LDAP and iterate on them. Each user's sync will start with +the following line that contains the user's username and email, as they +exist in GitLab now: + +```shell +Syncing user John, email@example.com +``` + +If you don't find a particular user's GitLab email in the output, then that +user hasn't logged in with LDAP yet. + +Next, GitLab searches its `identities` table for the existing +link between this user and the configured LDAP provider(s): + +```sql + Identity Load (0.9ms) SELECT "identities".* FROM "identities" WHERE "identities"."user_id" = 20 AND (provider LIKE 'ldap%') LIMIT 1 +``` + +The identity object will have the DN that GitLab will use to look for the user +in LDAP. If the DN isn't found, the email is used instead. We can see that +this user is found in LDAP: + +```shell +Instantiating Gitlab::Auth::Ldap::Person with LDIF: +dn: cn=John Smith,ou=people,dc=example,dc=com +cn: John Smith +mail: email@example.com +memberof: cn=admin_staff,ou=people,dc=example,dc=com +uid: John +``` + +If the user wasn't found in LDAP with either the DN or email, you may see the +following message instead: + +```shell +LDAP search error: No Such Object +``` + +...in which case the user will be blocked: + +```shell + User Update (0.4ms) UPDATE "users" SET "state" = $1, "updated_at" = $2 WHERE "users"."id" = $3 [["state", "ldap_blocked"], ["updated_at", "2019-10-18 15:46:22.902177"], ["id", 20]] +``` + +Once the user is found in LDAP the rest of the output will update the GitLab +database with any changes. + +#### Query a user in LDAP + +This will test that GitLab can reach out to LDAP and read a particular user. +It can expose potential errors connecting to and/or querying LDAP +that may seem to fail silently in the GitLab UI. + +```ruby +Rails.logger.level = Logger::DEBUG + +adapter = Gitlab::Auth::Ldap::Adapter.new('ldapmain') # If `main` is the LDAP provider +Gitlab::Auth::Ldap::Person.find_by_uid('<uid>', adapter) +``` + +### Group memberships **(STARTER ONLY)** + +#### Membership(s) not granted **(STARTER ONLY)** + +Sometimes you may think a particular user should be added to a GitLab group via +LDAP group sync, but for some reason it's not happening. There are several +things to check to debug the situation. + +- Ensure LDAP configuration has a `group_base` specified. + [This configuration](index.md#group-sync-starter-only) is required for group sync to work properly. +- Ensure the correct [LDAP group link is added to the GitLab + group](index.md#adding-group-links-starter-only). +- Check that the user has an LDAP identity: + 1. Sign in to GitLab as an administrator user. + 1. Navigate to **Admin area -> Users**. + 1. Search for the user + 1. Open the user, by clicking on their name. Do not click 'Edit'. + 1. Navigate to the **Identities** tab. There should be an LDAP identity with + an LDAP DN as the 'Identifier'. If not, this user hasn't logged in with + LDAP yet and must do so first. +- You've waited an hour or [the configured + interval](index.md#adjusting-ldap-group-sync-schedule-starter-only) for the group to + sync. To speed up the process, either go to the GitLab group **Settings -> + Members** and press **Sync now** (sync one group) or [run the group sync Rake + task](../../raketasks/ldap.md#run-a-group-sync-starter-only) (sync all groups). + +If all of the above looks good, jump in to a little more advanced debugging in +the rails console. + +1. Enter the [rails console](#rails-console). +1. Choose a GitLab group to test with. This group should have an LDAP group link + already configured. +1. [Enable debug logging, find the above GitLab group, and sync it with LDAP](#sync-one-group-starter-only). +1. Look through the output of the sync. See [example log + output](#example-console-output-after-a-group-sync-starter-only) + for how to read the output. +1. If you still aren't able to see why the user isn't being added, [query the + LDAP group directly](#query-a-group-in-ldap-starter-only) to see what members are listed. +1. Is the user's DN or UID in one of the lists from the above output? One of the DNs or + UIDs here should match the 'Identifier' from the LDAP identity checked earlier. If it doesn't, + the user does not appear to be in the LDAP group. + +#### Admin privileges not granted + +When [Administrator sync](index.md#administrator-sync-starter-only) has been configured +but the configured users aren't granted the correct admin privileges, confirm +the following are true: + +- A [`group_base` is also configured](index.md#group-sync-starter-only). +- The configured `admin_group` in the `gitlab.rb` is a CN, rather than a DN or an array. +- This CN falls under the scope of the configured `group_base`. +- The members of the `admin_group` have already logged into GitLab with their LDAP + credentials. GitLab will only grant this admin access to the users whose + accounts are already connected to LDAP. + +If all the above are true and the users are still not getting access, [run a manual +group sync](#sync-all-groups-starter-only) in the rails console and [look through the +output](#example-console-output-after-a-group-sync-starter-only) to see what happens when +GitLab syncs the `admin_group`. + +#### Sync all groups **(STARTER ONLY)** + +NOTE: **NOTE:** +To sync all groups manually when debugging is unnecessary, [use the Rake +task](../../raketasks/ldap.md#run-a-group-sync-starter-only) instead. + +The output from a manual [group sync](index.md#group-sync-starter-only) can show you what happens +when GitLab syncs its LDAP group memberships against LDAP. + +```ruby +Rails.logger.level = Logger::DEBUG + +LdapAllGroupsSyncWorker.new.perform +``` + +Next, [learn how to read the +output](#example-console-output-after-a-group-sync-starter-only). + +##### Example console output after a group sync **(STARTER ONLY)** + +Like the output from the user sync, the output from the [manual group +sync](#sync-all-groups-starter-only) will also be very verbose. However, it contains lots +of helpful information. + +Indicates the point where syncing actually begins: + +```shell +Started syncing 'ldapmain' provider for 'my_group' group +``` + +The following entry shows an array of all user DNs GitLab sees in the LDAP server. +Note that these are the users for a single LDAP group, not a GitLab group. If +you have multiple LDAP groups linked to this GitLab group, you will see multiple +log entries like this - one for each LDAP group. If you don't see an LDAP user +DN in this log entry, LDAP is not returning the user when we do the lookup. +Verify the user is actually in the LDAP group. + +```shell +Members in 'ldap_group_1' LDAP group: ["uid=john0,ou=people,dc=example,dc=com", +"uid=mary0,ou=people,dc=example,dc=com", "uid=john1,ou=people,dc=example,dc=com", +"uid=mary1,ou=people,dc=example,dc=com", "uid=john2,ou=people,dc=example,dc=com", +"uid=mary2,ou=people,dc=example,dc=com", "uid=john3,ou=people,dc=example,dc=com", +"uid=mary3,ou=people,dc=example,dc=com", "uid=john4,ou=people,dc=example,dc=com", +"uid=mary4,ou=people,dc=example,dc=com"] +``` + +Shortly after each of the above entries, you will see a hash of resolved member +access levels. This hash represents all user DNs GitLab thinks should have +access to this group, and at which access level (role). This hash is additive, +and more DNs may be added, or existing entries modified, based on additional +LDAP group lookups. The very last occurrence of this entry should indicate +exactly which users GitLab believes should be added to the group. + +NOTE: **Note:** +10 is 'Guest', 20 is 'Reporter', 30 is 'Developer', 40 is 'Maintainer' +and 50 is 'Owner'. + +```shell +Resolved 'my_group' group member access: {"uid=john0,ou=people,dc=example,dc=com"=>30, +"uid=mary0,ou=people,dc=example,dc=com"=>30, "uid=john1,ou=people,dc=example,dc=com"=>30, +"uid=mary1,ou=people,dc=example,dc=com"=>30, "uid=john2,ou=people,dc=example,dc=com"=>30, +"uid=mary2,ou=people,dc=example,dc=com"=>30, "uid=john3,ou=people,dc=example,dc=com"=>30, +"uid=mary3,ou=people,dc=example,dc=com"=>30, "uid=john4,ou=people,dc=example,dc=com"=>30, +"uid=mary4,ou=people,dc=example,dc=com"=>30} +``` + +It's not uncommon to see warnings like the following. These indicate that GitLab +would have added the user to a group, but the user could not be found in GitLab. +Usually this is not a cause for concern. + +If you think a particular user should already exist in GitLab, but you're seeing +this entry, it could be due to a mismatched DN stored in GitLab. See +[User DN and/or email have changed](#user-dn-orand-email-have-changed) to update the user's LDAP identity. + +```shell +User with DN `uid=john0,ou=people,dc=example,dc=com` should have access +to 'my_group' group but there is no user in GitLab with that +identity. Membership will be updated once the user signs in for +the first time. +``` + +Finally, the following entry says syncing has finished for this group: + +```shell +Finished syncing all providers for 'my_group' group +``` + +Once all the configured group links have been synchronized, GitLab will look +for any Administrators or External users to sync: + +```shell +Syncing admin users for 'ldapmain' provider +``` + +The output will look similar to what happens with a single group, and then +this line will indicate the sync is finished: + +```shell +Finished syncing admin users for 'ldapmain' provider +``` + +If [admin sync](index.md#administrator-sync-starter-only) is not configured, you'll see a message +stating as such: + +```shell +No `admin_group` configured for 'ldapmain' provider. Skipping +``` + +#### Sync one group **(STARTER ONLY)** + +[Syncing all groups](#sync-all-groups-starter-only) can produce a lot of noise in the output, which can be +distracting when you're only interested in troubleshooting the memberships of +a single GitLab group. In that case, here's how you can just sync this group +and see its debug output: + +```ruby +Rails.logger.level = Logger::DEBUG + +# Find the GitLab group. +# If the output is `nil`, the group could not be found. +# If a bunch of group attributes are in the output, your group was found successfully. +group = Group.find_by(name: 'my_gitlab_group') + +# Sync this group against LDAP +EE::Gitlab::Auth::Ldap::Sync::Group.execute_all_providers(group) +``` + +The output will be similar to +[that you'd get from syncing all groups](#example-console-output-after-a-group-sync-starter-only). + +#### Query a group in LDAP **(STARTER ONLY)** + +When you'd like to confirm that GitLab can read a LDAP group and see all its members, +you can run the following: + +```ruby +# Find the adapter and the group itself +adapter = Gitlab::Auth::Ldap::Adapter.new('ldapmain') # If `main` is the LDAP provider +ldap_group = EE::Gitlab::Auth::Ldap::Group.find_by_cn('group_cn_here', adapter) + +# Find the members of the LDAP group +ldap_group.member_dns +ldap_group.member_uids +``` + +### User DN or/and email have changed + +When an LDAP user is created in GitLab, their LDAP DN is stored for later reference. + +If GitLab cannot find a user by their DN, it will fall back +to finding the user by their email. If the lookup is successful, GitLab will +update the stored DN to the new value so both values will now match what's in +LDAP. + +If the email has changed and the DN has not, GitLab will find the user with +the DN and update its own record of the user's email to match the one in LDAP. + +However, if the primary email _and_ the DN change in LDAP, then GitLab will +have no way of identifying the correct LDAP record of the user and, as a +result, the user will be blocked. To rectify this, the user's existing +profile will have to be updated with at least one of the new values (primary +email or DN) so the LDAP record can be found. + +The following script will update the emails for all provided users so they +won't be blocked or unable to access their accounts. + +>**NOTE**: The following script will require that any new accounts with the new +email address are removed first. This is because emails have to be unique in GitLab. + +Go to the [rails console](#rails-console) and then run: + +```ruby +# Each entry will have to include the old username and the new email +emails = { + 'ORIGINAL_USERNAME' => 'NEW_EMAIL_ADDRESS', + ... +} + +emails.each do |username, email| + user = User.find_by_username(username) + user.email = email + user.skip_reconfirmation! + user.save! +end +``` + +You can then [run a UserSync](#sync-all-users-starter-only) **(STARTER ONLY)** to sync the latest DN +for each of these users. + +## Debugging Tools + +### LDAP check + +The [Rake task to check LDAP](../../raketasks/ldap.md#check) is a valuable tool +to help determine whether GitLab can successfully establish a connection to +LDAP and can get so far as to even read users. + +If a connection can't be established, it is likely either because of a problem +with your configuration or a firewall blocking the connection. + +- Ensure you don't have a firewall blocking the +connection, and that the LDAP server is accessible to the GitLab host. +- Look for an error message in the Rake check output, which may lead to your LDAP configuration to +confirm that the configuration values (specifically `host`, `port`, `bind_dn`, and +`password`) are correct. +- Look for [errors](#connection) in [the logs](#gitlab-logs) to further debug connection failures. + +If GitLab can successfully connect to LDAP but doesn't return any +users, [see what to do when no users are found](#no-users-are-found). + +### GitLab logs + +If a user account is blocked or unblocked due to the LDAP configuration, a +message will be [logged to `application.log`](../../logs.md#applicationlog). + +If there is an unexpected error during an LDAP lookup (configuration error, +timeout), the login is rejected and a message will be [logged to +`production.log`](../../logs.md#productionlog). + +### ldapsearch + +`ldapsearch` is a utility that will allow you to query your LDAP server. You can +use it to test your LDAP settings and ensure that the settings you're using +will get you the results you expect. + +When using `ldapsearch`, be sure to use the same settings you've already +specified in your `gitlab.rb` configuration so you can confirm what happens +when those exact settings are used. + +Running this command on the GitLab host will also help confirm that there's no +obstruction between the GitLab host and LDAP. + +For example, consider the following GitLab configuration: + +```shell +gitlab_rails['ldap_servers'] = YAML.load <<-'EOS' # remember to close this block with 'EOS' below + main: # 'main' is the GitLab 'provider ID' of this LDAP server + label: 'LDAP' + host: '127.0.0.1' + port: 389 + uid: 'uid' + encryption: 'plain' + bind_dn: 'cn=admin,dc=ldap-testing,dc=example,dc=com' + password: 'Password1' + active_directory: true + allow_username_or_email_login: false + block_auto_created_users: false + base: 'dc=ldap-testing,dc=example,dc=com' + user_filter: '' + attributes: + username: ['uid', 'userid', 'sAMAccountName'] + email: ['mail', 'email', 'userPrincipalName'] + name: 'cn' + first_name: 'givenName' + last_name: 'sn' + group_base: 'ou=groups,dc=ldap-testing,dc=example,dc=com' + admin_group: 'gitlab_admin' +EOS +``` + +You would run the following `ldapsearch` to find the `bind_dn` user: + +```shell +ldapsearch -D "cn=admin,dc=ldap-testing,dc=example,dc=com" \ + -w Password1 \ + -p 389 \ + -h 127.0.0.1 \ + -b "dc=ldap-testing,dc=example,dc=com" +``` + +Note that the `bind_dn`, `password`, `port`, `host`, and `base` are all +identical to what's configured in the `gitlab.rb`. + +Please see [the official +`ldapsearch` documentation](https://linux.die.net/man/1/ldapsearch) for more. + +### Using **AdFind** (Windows) + +You can use the [`AdFind`](https://social.technet.microsoft.com/wiki/contents/articles/7535.adfind-command-examples.aspx) utility (on Windows based systems) to test that your LDAP server is accessible and authentication is working correctly. This is a freeware utility built by [Joe Richards](http://www.joeware.net/freetools/tools/adfind/index.htm). + +**Return all objects** + +You can use the filter `objectclass=*` to return all directory objects. + +```shell +adfind -h ad.example.org:636 -ssl -u "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -up Password1 -b "OU=GitLab INT,DC=GitLab,DC=org" -f (objectClass=*) +``` + +**Return single object using filter** + +You can also retrieve a single object by **specifying** the object name or full **DN**. In this example we specify the object name only `CN=Leroy Fox`. + +```shell +adfind -h ad.example.org:636 -ssl -u "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -up Password1 -b "OU=GitLab INT,DC=GitLab,DC=org" -f (&(objectcategory=person)(CN=Leroy Fox))” +``` + +### Rails console + +CAUTION: **CAUTION:** +Please note that it is very easy to create, read, modify, and destroy data on the +rails console, so please be sure to run commands exactly as listed. + +The rails console is a valuable tool to help debug LDAP problems. It allows you to +directly interact with the application by running commands and seeing how GitLab +responds to them. + +Please refer to [this guide](../../troubleshooting/debug.md#starting-a-rails-console-session) +for instructions on how to use the rails console. + +#### Enable debug output + +This will provide debug output that will be useful to see +what GitLab is doing and with what. This value is not persisted, and will only +be enabled for this session in the rails console. + +To enable debug output in the rails console, [enter the rails +console](#rails-console) and run: + +```ruby +Rails.logger.level = Logger::DEBUG +``` diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index 7a808636e94..794733f860c 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -1,5 +1,8 @@ --- type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # OpenID Connect OmniAuth provider @@ -96,8 +99,8 @@ The OpenID Connect will provide you with a client details and secret for you to - `mtls` - Mutual TLS or X.509 certificate validation - Any other value will POST the client ID and secret in the request body - If not specified, defaults to `basic`. - - `<uid_field>` (optional) is the field name from the `user_info` details that will be used as `uid` value. For example, `preferred_username`. - If this value is not provided or the field with the configured value is missing from the `user_info` details, the `uid` will use the `sub` field. + - `<uid_field>` (optional) is the field name from the `user_info.raw_attributes` details that will be used as `uid` value. For example, `preferred_username`. + If this value is not provided or the field with the configured value is missing from the `user_info.raw_attributes` details, the `uid` will use the `sub` field. - `send_scope_to_token_endpoint` is `true` by default. In other words, the `scope` parameter is normally included in requests to the token endpoint. However, if your OpenID Connect provider does not accept the `scope` parameter in such requests, set this to `false`. - `client_options` are the OpenID Connect client-specific options. Specifically: diff --git a/doc/administration/auth/okta.md b/doc/administration/auth/okta.md index 4fd37b51f24..f7ab60ab56b 100644 --- a/doc/administration/auth/okta.md +++ b/doc/administration/auth/okta.md @@ -1,5 +1,8 @@ --- type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Okta SSO provider @@ -17,7 +20,7 @@ The following documentation enables Okta as a SAML provider. 1. Now, very important, add a logo (you can choose it from <https://about.gitlab.com/press/>). You'll have to crop and resize it. -1. Next, you'll need the to fill in the SAML general config. Here's an example +1. Next, you'll need the to fill in the SAML general configuration. Here's an example image.  @@ -25,14 +28,14 @@ The following documentation enables Okta as a SAML provider. 1. The last part of the configuration is the feedback section where you can just say you're a customer and creating an app for internal use. 1. When you have your app you'll have a few tabs on the top of the app's - profile. Click on the SAML 2.0 config instructions button which should + profile. Click on the SAML 2.0 configuration instructions button which should look like the following:  1. On the screen that comes up take note of the **Identity Provider Single Sign-On URL** which you'll use for the - `idp_sso_target_url` on your GitLab config file. + `idp_sso_target_url` on your GitLab configuration file. 1. **Before you leave Okta make sure you add your user and groups if any.** diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index 7131fd7571f..9ad1e0641f6 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -8,7 +8,7 @@ GitLab supports authentication using smartcards. ## Existing password authentication -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/33669) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33669) in GitLab 12.6. By default, existing users can continue to log in with a username and password when smartcard authentication is enabled. @@ -25,7 +25,7 @@ GitLab supports two authentication methods: ### Authentication against a local database with X.509 certificates -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/726) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.6 as an experimental feature. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/726) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.6 as an experimental feature. CAUTION: **Caution:** Smartcard authentication against local databases may change or be removed completely in future @@ -52,7 +52,7 @@ Certificate: ### Authentication against a local database with X.509 certificates and SAN extension -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8605) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8605) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. Smartcards with X.509 certificates using SAN extensions can be used to authenticate with GitLab. @@ -67,7 +67,7 @@ database with GitLab, in: - GitLab 12.4 and later, at least one of the `subjectAltName` (SAN) extensions need to define the user identity (`email`) within the GitLab instance (`URI`). `URI`: needs to match `Gitlab.config.host.gitlab`. -- From [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/33907), +- From [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/-/issues/33907), if your certificate contains only **one** SAN email entry, you don't need to add or modify it to match the `email` with the `URI`. @@ -95,7 +95,7 @@ Certificate: ### Authentication against an LDAP server -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7693) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8 as an experimental feature. Smartcard authentication against an LDAP server may change or be removed completely in future releases. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7693) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8 as an experimental feature. Smartcard authentication against an LDAP server may change or be removed completely in future releases. GitLab implements a standard way of certificate matching following [RFC4523](https://tools.ietf.org/html/rfc4523). It uses the @@ -159,7 +159,7 @@ attribute. As a prerequisite, you must use an LDAP server that: ``` For example, the following is an example server context in an NGINX - configuration file (eg. in `/etc/nginx/sites-available/gitlab-ssl`): + configuration file (such as in `/etc/nginx/sites-available/gitlab-ssl`): ```plaintext server { diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 8cd10f5ea4e..47e4b6c068c 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -12,8 +12,8 @@ GitLab’s [security features](../security/README.md) may also help you meet rel |**[Email all users of a project, group, or entire server](../user/admin_area/settings/terms.md)**<br>An admin can email groups of users based on project or group membership, or email everyone using the GitLab instance. This is great for scheduled maintenance or upgrades.|Starter+|| |**[Omnibus package supports log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-forwarding)**<br>Forward your logs to a central system.|Starter+|| |**[Lock project membership to group](../user/group/index.md#member-lock-starter)**<br>Group owners can prevent new members from being added to projects within a group.|Starter+|✓| -|**[LDAP group sync](auth/ldap-ee.md#group-sync)**<br>GitLab Enterprise Edition gives admins the ability to automatically sync groups and manage SSH keys, permissions, and authentication, so you can focus on building your product, not configuring your tools.|Starter+|| -|**[LDAP group sync filters](auth/ldap-ee.md#group-sync)**<br>GitLab Enterprise Edition Premium gives more flexibility to synchronize with LDAP based on filters, meaning you can leverage LDAP attributes to map GitLab permissions.|Premium+|| +|**[LDAP group sync](auth/ldap/index.md#group-sync-starter-only)**<br>GitLab Enterprise Edition gives admins the ability to automatically sync groups and manage SSH keys, permissions, and authentication, so you can focus on building your product, not configuring your tools.|Starter+|| +|**[LDAP group sync filters](auth/ldap/index.md#group-sync-starter-only)**<br>GitLab Enterprise Edition Premium gives more flexibility to synchronize with LDAP based on filters, meaning you can leverage LDAP attributes to map GitLab permissions.|Premium+|| |**[Audit logs](audit_events.md)**<br>To maintain the integrity of your code, GitLab Enterprise Edition Premium gives admins the ability to view any modifications made within the GitLab server in an advanced audit log system, so you can control, analyze, and track every change.|Premium+|| |**[Auditor users](auditor_users.md)**<br>Auditor users are users who are given read-only access to all projects, groups, and other resources on the GitLab instance.|Premium+|| |**[Credentials inventory](../user/admin_area/credentials_inventory.md)**<br>With a credentials inventory, GitLab administrators can keep track of the credentials used by all of the users in their GitLab instance. |Ultimate|| diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index 37d7194af53..ecf9464eb91 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -1,3 +1,10 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + # Environment Variables GitLab exposes certain environment variables which can be used to override @@ -63,6 +70,7 @@ to the naming scheme `GITLAB_#{name in 1_settings.rb in upper case}`. To set environment variables, follow [these instructions](https://docs.gitlab.com/omnibus/settings/environment-variables.html). -It's possible to preconfigure the GitLab docker image by adding the environment +It's possible to preconfigure the GitLab Docker image by adding the environment variable `GITLAB_OMNIBUS_CONFIG` to the `docker run` command. -For more information see the ['preconfigure-docker-container' section in the Omnibus documentation](https://docs.gitlab.com/omnibus/docker/#preconfigure-docker-container). +For more information see the [Pre-configure Docker container](https://docs.gitlab.com/omnibus/docker/#pre-configure-docker-container) +section in the Omnibus documentation. diff --git a/doc/administration/external_database.md b/doc/administration/external_database.md index 47509828c20..7424b4e1e83 100644 --- a/doc/administration/external_database.md +++ b/doc/administration/external_database.md @@ -1,41 +1,7 @@ -# Configure GitLab using an external PostgreSQL service - -If you're hosting GitLab on a cloud provider, you can optionally use a -managed service for PostgreSQL. For example, AWS offers a managed Relational -Database Service (RDS) that runs PostgreSQL. - -Alternatively, you may opt to manage your own PostgreSQL instance or cluster -separate from the Omnibus GitLab package. - -If you use a cloud-managed service, or provide your own PostgreSQL instance: - -1. Set up PostgreSQL according to the - [database requirements document](../install/requirements.md#database). -1. Set up a `gitlab` username with a password of your choice. The `gitlab` user - needs privileges to create the `gitlabhq_production` database. -1. If you are using a cloud-managed service, you may need to grant additional - roles to your `gitlab` user: - - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. - - Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. +--- +redirect_to: 'postgresql/external.md' +--- -1. Configure the GitLab application servers with the appropriate connection details - for your external PostgreSQL service in your `/etc/gitlab/gitlab.rb` file: - - ```ruby - # Disable the bundled Omnibus provided PostgreSQL - postgresql['enable'] = false - - # PostgreSQL connection details - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' - gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server - gitlab_rails['db_password'] = 'DB password' - ``` - - For more information on GitLab HA setups, refer to [configuring GitLab for HA](high_availability/gitlab.md). - -1. Reconfigure for the changes to take effect: +# Configure GitLab using an external PostgreSQL service - ```shell - sudo gitlab-ctl reconfigure - ``` +This content has been moved to a [new location](postgresql/external.md). diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md index 489d1924803..cb1d35f24d5 100644 --- a/doc/administration/external_pipeline_validation.md +++ b/doc/administration/external_pipeline_validation.md @@ -1,3 +1,10 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + # External Pipeline Validation You can use an external service for validating a pipeline before it's created. diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index 59cd5497032..9e87eed4508 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference description: "GitLab administrator: enable and disable GitLab features deployed behind feature flags" --- @@ -29,7 +32,7 @@ them. It can be done by GitLab administrators with access to GitLab Rails console. If you used a certain feature and identified a bug, a misbehavior, or an -error, it's very important that you [**provide feedback**](https://gitlab.com/gitlab-org/gitlab/issues/new?issue[title]=Docs%20-%20feature%20flag%20feedback%3A%20Feature%20Name&issue[description]=Describe%20the%20problem%20you%27ve%20encountered.%0A%0A%3C!--%20Don%27t%20edit%20below%20this%20line%20--%3E%0A%0A%2Flabel%20~%22docs%5C-comments%22%20) to GitLab as soon +error, it's very important that you [**provide feedback**](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue[title]=Docs%20-%20feature%20flag%20feedback%3A%20Feature%20Name&issue[description]=Describe%20the%20problem%20you%27ve%20encountered.%0A%0A%3C!--%20Don%27t%20edit%20below%20this%20line%20--%3E%0A%0A%2Flabel%20~%22docs%5C-comments%22%20) to GitLab as soon as possible so we can improve or fix it while behind a flag. When you upgrade GitLab to an earlier version, the feature flag status may change. @@ -94,6 +97,18 @@ Example, to disable Evidence Collection: Feature.disable(:release_evidence_collection) ``` +Some feature flags can be enabled or disabled on a per project basis: + +```ruby +Feature.enable(:<feature flag>, Project.find(<project id>)) +``` + +For example, to enable the [`:release_evidence_collection`](../ci/junit_test_reports.md#enabling-the-feature) feature flag for project `1234`: + +```ruby +Feature.enable(:release_evidence_collection, Project.find(1234)) +``` + When the feature is ready, GitLab will remove the feature flag, the option for enabling and disabling it will no longer exist, and the feature will become available in all instances. diff --git a/doc/administration/geo/disaster_recovery/background_verification.md b/doc/administration/geo/disaster_recovery/background_verification.md index 5b24c222f06..88218d24b2f 100644 --- a/doc/administration/geo/disaster_recovery/background_verification.md +++ b/doc/administration/geo/disaster_recovery/background_verification.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Automatic background verification **(PREMIUM ONLY)** NOTE: **Note:** diff --git a/doc/administration/geo/disaster_recovery/bring_primary_back.md b/doc/administration/geo/disaster_recovery/bring_primary_back.md index 43089237a75..b19e55595e7 100644 --- a/doc/administration/geo/disaster_recovery/bring_primary_back.md +++ b/doc/administration/geo/disaster_recovery/bring_primary_back.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Bring a demoted primary node back online **(PREMIUM ONLY)** After a failover, it is possible to fail back to the demoted **primary** node to diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 9f7afad353b..f690ec63cf9 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Disaster Recovery (Geo) **(PREMIUM ONLY)** Geo replicates your database, your Git repositories, and few other assets. @@ -51,7 +58,7 @@ must disable the **primary** node. NOTE: **Note:** (**CentOS only**) In CentOS 6 or older, there is no easy way to prevent GitLab from being - started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3058)). + started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). It may be safest to uninstall the GitLab package completely: ```shell @@ -60,7 +67,7 @@ must disable the **primary** node. NOTE: **Note:** (**Ubuntu 14.04 LTS**) If you are using an older version of Ubuntu - or any other distro based on the Upstart init system, you can prevent GitLab + or any other distribution based on the Upstart init system, you can prevent GitLab from starting if the machine reboots by doing the following: ```shell @@ -121,14 +128,20 @@ Note the following when promoting a secondary: gitlab-ctl promote-to-primary-node ``` + If you have already run the [preflight checks](planned_failover.md#preflight-checks), you can skip them with: + + ```shell + gitlab-ctl promote-to-primary-node --skip-preflight-check + ``` + 1. Verify you can connect to the newly promoted **primary** node using the URL used previously for the **secondary** node. 1. If successful, the **secondary** node has now been promoted to the **primary** node. -#### Promoting a **secondary** node with HA +#### Promoting a **secondary** node with multiple servers The `gitlab-ctl promote-to-primary-node` command cannot be used yet in -conjunction with High Availability or with multiple machines, as it can only +conjunction with multiple servers, as it can only perform changes on a **secondary** with only a single machine. Instead, you must do this manually. @@ -178,6 +191,27 @@ required: set the database to read-write: - Amazon RDS - [Promoting a Read Replica to Be a Standalone DB Instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Promote) - Azure Database for PostgreSQL - [Stop replication](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal#stop-replication) + - Other external PostgreSQL databases - save the script below in you secondary node, for example + `/tmp/geo_promote.sh`, and modify the connection parameters to match your + environment. Then, execute it to promote the replica: + + ```shell + #!/bin/bash + + PG_SUPERUSER=postgres + + # The path to your pg_ctl binary. You may need to adjust this path to match + # your PostgreSQL installation + PG_CTL_BINARY=/usr/lib/postgresql/10/bin/pg_ctl + + # The path to your PostgreSQL data directory. You may need to adjust this + # path to match your PostgreSQL installation. You can also run + # `SHOW data_directory;` from PostgreSQL to find your data directory + PG_DATA_DIRECTORY=/etc/postgresql/10/main + + # Promote the PostgreSQL database and allow read/write operations + sudo -u $PG_SUPERUSER $PG_CTL_BINARY -D $PG_DATA_DIRECTORY promote + ``` 1. Edit `/etc/gitlab/gitlab.rb` on every node in the **secondary** site to reflect its new status as **primary** by removing any lines that enabled the diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 00ca1e4b8b0..0ce1325a537 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Disaster recovery for planned failover **(PREMIUM ONLY)** The primary use-case of Disaster Recovery is to ensure business continuity in @@ -126,7 +133,7 @@ will take to finish syncing. An example message would be: ## Prevent updates to the **primary** node -Until a [read-only mode](https://gitlab.com/gitlab-org/gitlab-foss/issues/19739) is implemented, updates must be prevented +Until a [read-only mode](https://gitlab.com/gitlab-org/gitlab/-/issues/14609) is implemented, updates must be prevented from happening manually. Note that your **secondary** node still needs read-only access to the **primary** node during the maintenance window. diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 86a8e5b28d1..3d08ed81700 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo configuration **(PREMIUM ONLY)** ## Configuring a new **secondary** node @@ -25,7 +32,7 @@ Any change that requires access to the **Admin Area** needs to be done in the GitLab stores a number of secret values in the `/etc/gitlab/gitlab-secrets.json` file which *must* be the same on all nodes. Until there is -a means of automatically replicating these between nodes (see [issue #3789](https://gitlab.com/gitlab-org/gitlab/issues/3789)), +a means of automatically replicating these between nodes (see [issue #3789](https://gitlab.com/gitlab-org/gitlab/-/issues/3789)), they must be manually replicated to the **secondary** node. 1. SSH into the **primary** node, and execute the command below: diff --git a/doc/administration/geo/replication/database.md b/doc/administration/geo/replication/database.md index 62bd0e6ac19..3f2d46ba457 100644 --- a/doc/administration/geo/replication/database.md +++ b/doc/administration/geo/replication/database.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo database replication **(PREMIUM ONLY)** NOTE: **Note:** @@ -23,10 +30,6 @@ The GitLab **primary** node where the write operations happen will connect to the **primary** database server, and **secondary** nodes will connect to their own database servers (which are also read-only). -NOTE: **Note:** -In database documentation, you may see "**primary**" being referenced as "master" -and "**secondary**" as either "slave" or "standby" server (read-only). - We recommend using [PostgreSQL replication slots](https://medium.com/@tk512/replication-slots-in-postgresql-b4b03d277c75) to ensure that the **primary** node retains all the data necessary for the **secondary** nodes to recover. See below for more details. @@ -43,7 +46,7 @@ The following guide assumes that: CAUTION: **Warning:** Geo works with streaming replication. Logical replication is not supported at this time. -There is an [issue where support is being discussed](https://gitlab.com/gitlab-org/gitlab/issues/7420). +There is an [issue where support is being discussed](https://gitlab.com/gitlab-org/gitlab/-/issues/7420). ### Step 1. Configure the **primary** server @@ -317,7 +320,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o to the private key, which is **only** present on the **primary** node. 1. Test that the `gitlab-psql` user can connect to the **primary** node's database - (the default Omnibus database name is gitlabhq_production): + (the default Omnibus database name is `gitlabhq_production`): ```shell sudo \ @@ -461,7 +464,7 @@ The replication process is now complete. PostgreSQL connections. We recommend using PgBouncer if you use GitLab in a high-availability configuration with a cluster of nodes supporting a Geo **primary** node and another cluster of nodes supporting a Geo **secondary** node. For more -information, see [High Availability with Omnibus GitLab](../../high_availability/database.md#high-availability-with-omnibus-gitlab-premium-only). +information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). For a Geo **secondary** node to work properly with PgBouncer in front of the database, it will need a separate read-only user to make [PostgreSQL FDW queries](https://www.postgresql.org/docs/11/postgres-fdw.html) diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 17031b11f51..f50da27d82f 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo data types support A Geo data type is a specific class of data that is required by one or more GitLab features to @@ -129,11 +136,11 @@ successfully, you must replicate their data using some other means. | Application data in PostgreSQL | **Yes** | **Yes** | | | Project repository | **Yes** | **Yes** | | | Project wiki repository | **Yes** | **Yes** | | -| Project designs repository | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/issues/32467) | | +| Project designs repository | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | | | Uploads | **Yes** | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | Verified only on transfer, or manually (*1*) | -| LFS objects | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/issues/8922) | Verified only on transfer, or manually (*1*). Unavailable for new LFS objects in 11.11.x and 12.0.x (*2*). | -| CI job artifacts (other than traces) | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/issues/8923) | Verified only manually (*1*) | -| Archived traces | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/issues/8923) | Verified only on transfer, or manually (*1*) | +| LFS objects | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8922) | Verified only on transfer, or manually (*1*). Unavailable for new LFS objects in 11.11.x and 12.0.x (*2*). | +| CI job artifacts (other than traces) | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only manually (*1*) | +| Archived traces | **Yes** | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Verified only on transfer, or manually (*1*) | | Personal snippets | **Yes** | **Yes** | | | [Versioned snippets](../../../user/snippets.md#versioned-snippets) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2810) | | | Project snippets | **Yes** | **Yes** | | @@ -147,11 +154,16 @@ successfully, you must replicate their data using some other means. | [Conan Repository](../../../user/packages/conan_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | | [NuGet Repository](../../../user/packages/nuget_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2346) | No | | | [PyPi Repository](../../../user/packages/pypi_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2554) | No | | -| [External merge request diffs](../../merge_request_diffs.md) | [No](https://gitlab.com/gitlab-org/gitlab/issues/33817) | No | | +| [Composer Repository](../../../user/packages/composer_repository/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3096) | No | | +| [External merge request diffs](../../merge_request_diffs.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/33817) | No | | +| [Terraform State](../../terraform_state.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3112)(*3*) | No | | +| [Vulnerability Export](../../../user/application_security/security_dashboard/#export-vulnerabilities-1) | [No](https://gitlab.com/groups/gitlab-org/-/epics/3111)(*3*) | No | | | | Content in object storage | **Yes** | No | | - (*1*): The integrity can be verified manually using [Integrity Check Rake Task](../../raketasks/check.md) on both nodes and comparing the output between them. - (*2*): GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new - LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/issues/32696). + LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). +- (*3*): If you are using Object Storage, the replication can be performed by the + Object Storage provider if supported. Please see [Geo with Object Storage](object_storage.md) diff --git a/doc/administration/geo/replication/docker_registry.md b/doc/administration/geo/replication/docker_registry.md index e9c69dd32e4..bea6528dc9b 100644 --- a/doc/administration/geo/replication/docker_registry.md +++ b/doc/administration/geo/replication/docker_registry.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Docker Registry for a secondary node **(PREMIUM ONLY)** You can set up a [Docker Registry](https://docs.docker.com/registry/) on your @@ -5,7 +12,7 @@ You can set up a [Docker Registry](https://docs.docker.com/registry/) on your ## Storage support -Docker Registry currently supports a few types of storages. If you choose a +Docker Registry currently supports a few types of storage. If you choose a distributed storage (`azure`, `gcs`, `s3`, `swift`, or `oss`) for your Docker Registry on the **primary** node, you can use the same storage for a **secondary** Docker Registry as well. For more information, read the @@ -16,7 +23,7 @@ integrated [Container Registry](../../packages/container_registry.md#container-r ## Replicating Docker Registry You can enable a storage-agnostic replication so it -can be used for cloud or local storages. Whenever a new image is pushed to the +can be used for cloud or local storage. Whenever a new image is pushed to the **primary** node, each **secondary** node will pull it to its own container repository. @@ -104,7 +111,7 @@ generate a short-lived JWT that is pull-only-capable to access the ```ruby gitlab_rails['geo_registry_replication_enabled'] = true - gitlab_rails['geo_registry_replication_primary_api_url'] = 'http://primary.example.com:4567/' # Primary registry address, it will be used by the secondary node to directly communicate to primary registry + gitlab_rails['geo_registry_replication_primary_api_url'] = 'https://primary.example.com:5050/' # Primary registry address, it will be used by the secondary node to directly communicate to primary registry ``` 1. Reconfigure the **secondary** node for the change to take effect: diff --git a/doc/administration/geo/replication/external_database.md b/doc/administration/geo/replication/external_database.md index ae3069a0e40..491b3278ead 100644 --- a/doc/administration/geo/replication/external_database.md +++ b/doc/administration/geo/replication/external_database.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo with external PostgreSQL instances **(PREMIUM ONLY)** This document is relevant if you are using a PostgreSQL instance that is *not @@ -62,7 +69,7 @@ Once your read-only replica is set up, you can skip to [configure you secondary #### Manually configure the primary database for replication -The [geo_primary_role](https://docs.gitlab.com/omnibus/roles/#gitlab-geo-roles) +The [`geo_primary_role`](https://docs.gitlab.com/omnibus/roles/#gitlab-geo-roles) configures the **primary** node's database to be replicated by making changes to `pg_hba.conf` and `postgresql.conf`. Make the following configuration changes manually to your external database configuration and ensure that you restart PostgreSQL @@ -123,7 +130,7 @@ hot_standby = on ### Configure **secondary** application nodes to use the external read-replica With Omnibus, the -[geo_secondary_role](https://docs.gitlab.com/omnibus/roles/#gitlab-geo-roles) +[`geo_secondary_role`](https://docs.gitlab.com/omnibus/roles/#gitlab-geo-roles) has three main functions: 1. Configure the replica database. diff --git a/doc/administration/geo/replication/faq.md b/doc/administration/geo/replication/faq.md index 2405e2cbfd2..522ad32c352 100644 --- a/doc/administration/geo/replication/faq.md +++ b/doc/administration/geo/replication/faq.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo Frequently Asked Questions **(PREMIUM ONLY)** ## What are the minimum requirements to run Geo? diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index a8b0bdeb7da..7b186d15fae 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo validation tests The Geo team performs manual testing and validation on common deployment configurations to ensure @@ -26,8 +33,8 @@ The following are GitLab upgrade validation tests we performed. configuration. - Outcome: Upgrade test was successful. - Follow up issues: - - [Investigate Geo end-to-end test failures](https://gitlab.com/gitlab-org/gitlab/issues/201823). - - [Add more logging to Geo end-to-end tests](https://gitlab.com/gitlab-org/gitlab/issues/201830). + - [Investigate Geo end-to-end test failures](https://gitlab.com/gitlab-org/gitlab/-/issues/201823). + - [Add more logging to Geo end-to-end tests](https://gitlab.com/gitlab-org/gitlab/-/issues/201830). - [Excess service restarts during zero-downtime upgrade](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5047). [Upgrade Geo multi-server installation](https://gitlab.com/gitlab-org/gitlab/-/issues/199836): @@ -43,7 +50,7 @@ The following are GitLab upgrade validation tests we performed. configuration. - Outcome: Upgrade test was successful. - Follow up issues: - - [Investigate why HTTP push spec failed on primary node](https://gitlab.com/gitlab-org/gitlab/issues/199825). + - [Investigate why HTTP push spec failed on primary node](https://gitlab.com/gitlab-org/gitlab/-/issues/199825). - [Investigate if documentation should be modified to include refresh foreign tables task](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5041). ### October 2019 @@ -89,7 +96,7 @@ The following are PostgreSQL upgrade validation tests we performed. ### September 2019 -[Test and validate PostgreSQL 10.0 upgrade for Geo](https://gitlab.com/gitlab-org/gitlab/issues/12092): +[Test and validate PostgreSQL 10.0 upgrade for Geo](https://gitlab.com/gitlab-org/gitlab/-/issues/12092): - Description: With the 12.0 release, GitLab required an upgrade to PostgreSQL 10.0. We tested various upgrade scenarios from GitLab 11.11.5 through to GitLab 12.1.8. diff --git a/doc/administration/geo/replication/index.md b/doc/administration/geo/replication/index.md index 87bd7b69515..5b4b476bfa8 100644 --- a/doc/administration/geo/replication/index.md +++ b/doc/administration/geo/replication/index.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Replication (Geo) **(PREMIUM ONLY)** > - Introduced in GitLab Enterprise Edition 8.9. @@ -191,7 +198,7 @@ If you installed GitLab using the Omnibus packages (highly recommended): 1. [Set up the database replication](database.md) (`primary (read-write) <-> secondary (read-only)` topology). 1. [Configure fast lookup of authorized SSH keys in the database](../../operations/fast_ssh_key_lookup.md). This step is required and needs to be done on **both** the **primary** and **secondary** nodes. 1. [Configure GitLab](configuration.md) to set the **primary** and **secondary** nodes. -1. Optional: [Configure a secondary LDAP server](../../auth/ldap.md) for the **secondary** node. See [notes on LDAP](#ldap). +1. Optional: [Configure a secondary LDAP server](../../auth/ldap/index.md) for the **secondary** node. See [notes on LDAP](#ldap). 1. [Follow the "Using a Geo Server" guide](using_a_geo_server.md). ## Post-installation documentation @@ -243,15 +250,15 @@ For more information on removing a Geo node, see [Removing **secondary** Geo nod CAUTION: **Caution:** This list of limitations only reflects the latest version of GitLab. If you are using an older version, extra limitations may be in place. -- Pushing directly to a **secondary** node redirects (for HTTP) or proxies (for SSH) the request to the **primary** node instead of [handling it directly](https://gitlab.com/gitlab-org/gitlab/issues/1381), except when using Git over HTTP with credentials embedded within the URI. For example, `https://user:password@secondary.tld`. +- Pushing directly to a **secondary** node redirects (for HTTP) or proxies (for SSH) the request to the **primary** node instead of [handling it directly](https://gitlab.com/gitlab-org/gitlab/-/issues/1381), except when using Git over HTTP with credentials embedded within the URI. For example, `https://user:password@secondary.tld`. - Cloning, pulling, or pushing repositories that exist on the **primary** node but not on the **secondary** nodes where [selective synchronization](configuration.md#selective-synchronization) does not include the project is not supported over SSH [but support is planned](https://gitlab.com/groups/gitlab-org/-/epics/2562). HTTP(S) is supported. -- The **primary** node has to be online for OAuth login to happen. Existing sessions and Git are not affected. Support for the **secondary** node to use an OAuth provider independent from the primary is [being planned](https://gitlab.com/gitlab-org/gitlab/issues/208465). -- The installation takes multiple manual steps that together can take about an hour depending on circumstances. We are working on improving this experience. See [Omnibus GitLab issue #2978](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2978) for details. +- The **primary** node has to be online for OAuth login to happen. Existing sessions and Git are not affected. Support for the **secondary** node to use an OAuth provider independent from the primary is [being planned](https://gitlab.com/gitlab-org/gitlab/-/issues/208465). +- The installation takes multiple manual steps that together can take about an hour depending on circumstances. We are working on improving this experience. See [Omnibus GitLab issue #2978](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/2978) for details. - Real-time updates of issues/merge requests (for example, via long polling) doesn't work on the **secondary** node. - [Selective synchronization](configuration.md#selective-synchronization) applies only to files and repositories. Other datasets are replicated to the **secondary** node in full, making it inappropriate for use as an access control mechanism. - Object pools for forked project deduplication work only on the **primary** node, and are duplicated on the **secondary** node. - [External merge request diffs](../../merge_request_diffs.md) will not be replicated if they are on-disk, and viewing merge requests will fail. However, external MR diffs in object storage **are** supported. The default configuration (in-database) does work. -- GitLab Runners cannot register with a **secondary** node. Support for this is [planned for the future](https://gitlab.com/gitlab-org/gitlab/issues/3294). +- GitLab Runners cannot register with a **secondary** node. Support for this is [planned for the future](https://gitlab.com/gitlab-org/gitlab/-/issues/3294). ### Limitations on replication/verification diff --git a/doc/administration/geo/replication/location_aware_git_url.md b/doc/administration/geo/replication/location_aware_git_url.md index f1f1edd4a9b..49c83ee1718 100644 --- a/doc/administration/geo/replication/location_aware_git_url.md +++ b/doc/administration/geo/replication/location_aware_git_url.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Location-aware Git remote URL with AWS Route53 **(PREMIUM ONLY)** You can provide GitLab users with a single remote URL that automatically uses @@ -15,7 +22,7 @@ NOTE: **Note** You can also use a load balancer to distribute web UI or API traffic to [multiple Geo **secondary** nodes](../../../user/admin_area/geo_nodes.md#multiple-secondary-nodes-behind-a-load-balancer). Importantly, the **primary** node cannot yet be included. See the feature request -[Support putting the **primary** behind a Geo node load balancer](https://gitlab.com/gitlab-org/gitlab/issues/10888). +[Support putting the **primary** behind a Geo node load balancer](https://gitlab.com/gitlab-org/gitlab/-/issues/10888). ## Prerequisites diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index 9322c4cc417..2747aaeb105 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo for multiple servers **(PREMIUM ONLY)** This document describes a minimal reference architecture for running Geo @@ -38,7 +45,7 @@ Because of the additional complexity involved in setting up this configuration for PostgreSQL and Redis, it is not covered by this Geo multi-server documentation. For more information about setting up a multi-server PostgreSQL cluster and Redis cluster using the omnibus package see the multi-server documentation for -[PostgreSQL](../../high_availability/database.md) and +[PostgreSQL](../../postgresql/replication_and_failover.md) and [Redis](../../high_availability/redis.md), respectively. NOTE: **Note:** @@ -87,7 +94,7 @@ NOTE: **Note:** PostgreSQL and Redis should have already been disabled on the application servers, and connections from the application servers to those services on the backend servers configured, during normal GitLab multi-server set up. See multi-server configuration documentation for -[PostgreSQL](../../high_availability/database.md#configuring-the-application-nodes) +[PostgreSQL](../../postgresql/replication_and_failover.md#configuring-the-application-nodes) and [Redis](../../high_availability/redis.md#example-configuration-for-the-gitlab-application). ### Step 2: Configure the **primary** database diff --git a/doc/administration/geo/replication/object_storage.md b/doc/administration/geo/replication/object_storage.md index ffd44282b23..3cc0ade414e 100644 --- a/doc/administration/geo/replication/object_storage.md +++ b/doc/administration/geo/replication/object_storage.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo with Object storage **(PREMIUM ONLY)** Geo can be used in combination with Object Storage (AWS S3, or other compatible object storage). @@ -16,7 +23,7 @@ To have: ## Enabling GitLab managed object storage replication -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10586) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10586) in GitLab 12.4. CAUTION: **Caution:** This is a [**beta** feature](https://about.gitlab.com/handbook/product/#beta) and is not ready yet for production use at any scale. diff --git a/doc/administration/geo/replication/remove_geo_node.md b/doc/administration/geo/replication/remove_geo_node.md index c04c7aec858..539132776b3 100644 --- a/doc/administration/geo/replication/remove_geo_node.md +++ b/doc/administration/geo/replication/remove_geo_node.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Removing secondary Geo nodes **(PREMIUM ONLY)** **Secondary** nodes can be removed from the Geo cluster using the Geo admin page of the **primary** node. To remove a **secondary** node: diff --git a/doc/administration/geo/replication/security_review.md b/doc/administration/geo/replication/security_review.md index 0ac8157220a..f5edf79c6e4 100644 --- a/doc/administration/geo/replication/security_review.md +++ b/doc/administration/geo/replication/security_review.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo security review (Q&A) **(PREMIUM ONLY)** The following security review of the Geo feature set focuses on security aspects of @@ -22,7 +29,7 @@ from [owasp.org](https://owasp.org/). etc) and repository + wiki data. In a typical configuration, this will happen across the public Internet, and be TLS-encrypted. - PostgreSQL replication is TLS-encrypted. -- See also: [only TLSv1.2 should be supported](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2948) +- See also: [only TLSv1.2 should be supported](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/2948) ### How can the data be classified into categories according to its sensitivity? @@ -72,7 +79,7 @@ from [owasp.org](https://owasp.org/). - Nothing Geo-specific. Any user where `admin: true` is set in the database is considered an admin with super-user privileges. -- See also: [more granular access control](https://gitlab.com/gitlab-org/gitlab-foss/issues/32730) +- See also: [more granular access control](https://gitlab.com/gitlab-org/gitlab/-/issues/18242) (not Geo-specific). - Much of Geo’s integration (database replication, for instance) must be configured with the application, typically by system administrators. diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 293414a6e5e..b03a2dae971 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Geo Troubleshooting **(PREMIUM ONLY)** Setting up Geo requires careful attention to details and sometimes it's easy to @@ -332,7 +339,7 @@ some of these queries will never be able to complete due to being canceled every time. These long-running queries are -[planned to be removed in the future](https://gitlab.com/gitlab-org/gitlab/issues/34269), +[planned to be removed in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/34269), but as a workaround, we recommend enabling [hot_standby_feedback](https://www.postgresql.org/docs/10/hot-standby.html#HOT-STANDBY-CONFLICT). This increases the likelihood of bloat on the **primary** node as it prevents @@ -353,7 +360,7 @@ sudo gitlab-ctl reconfigure ``` To help us resolve this problem, consider commenting on -[the issue](https://gitlab.com/gitlab-org/gitlab/issues/4489). +[the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/4489). ### Message: `LOG: invalid CIDR mask in address` @@ -405,7 +412,7 @@ long enough to accommodate a full clone of your largest repositories. If new LFS objects are never replicated to secondary Geo nodes, check the version of GitLab you are running. GitLab versions 11.11.x or 12.0.x are affected by -[a bug that results in new LFS objects not being replicated to Geo secondary nodes](https://gitlab.com/gitlab-org/gitlab/issues/32696). +[a bug that results in new LFS objects not being replicated to Geo secondary nodes](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). To resolve the issue, upgrade to GitLab 12.1 or newer. @@ -535,7 +542,7 @@ or `gitlab-ctl promote-to-primary-node`, either: ```ruby Rails.application.load_tasks; nil - Gitlab::Geo.expire_cache_keys!([:primary_node, :current_node]) + Gitlab::Geo.expire_cache! Rake::Task['geo:set_secondary_as_primary'].invoke ``` @@ -568,7 +575,7 @@ is displayed if you attempt to run this command on a primary node. ### Message: `sudo: gitlab-pg-ctl: command not found` When -[promoting a **secondary** node with HA](../disaster_recovery/index.md#promoting-a-secondary-node-with-ha), +[promoting a **secondary** node with multiple servers](../disaster_recovery/index.md#promoting-a-secondary-node-with-multiple-servers), you need to run the `gitlab-pg-ctl` command to promote the PostgreSQL read-replica database. diff --git a/doc/administration/geo/replication/tuning.md b/doc/administration/geo/replication/tuning.md index 972bf002935..63a8f81eecb 100644 --- a/doc/administration/geo/replication/tuning.md +++ b/doc/administration/geo/replication/tuning.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Tuning Geo **(PREMIUM ONLY)** ## Changing the sync capacity values diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index fa1576e19eb..6c2778ad0fe 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Updating the Geo nodes **(PREMIUM ONLY)** Updating Geo nodes involves performing: diff --git a/doc/administration/geo/replication/using_a_geo_server.md b/doc/administration/geo/replication/using_a_geo_server.md index 2fec2b2b59c..3f2895f1c71 100644 --- a/doc/administration/geo/replication/using_a_geo_server.md +++ b/doc/administration/geo/replication/using_a_geo_server.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + <!-- Please update EE::GitLab::GeoGitAccess::GEO_SERVER_DOCS_URL if this file is moved) --> # Using a Geo Server **(PREMIUM ONLY)** diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md index db8bbddec3b..777de715a8c 100644 --- a/doc/administration/geo/replication/version_specific_updates.md +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -1,3 +1,10 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Version specific update instructions Check this document if it includes instructions for the version you are updating. @@ -16,7 +23,7 @@ The issue is fixed in GitLab 12.9.4. Please upgrade to GitLab 12.9.4 or later. DANGER: **Danger:** Only upgrade to GitLab 12.7.5 or later. Do not upgrade to versions 12.7.0 through 12.7.4 because there is [an initialization order -bug](https://gitlab.com/gitlab-org/gitlab/issues/199672) that causes Geo +bug](https://gitlab.com/gitlab-org/gitlab/-/issues/199672) that causes Geo **secondaries** to set the incorrect database connection pool size. [The fix](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24021) was shipped in 12.7.5. @@ -56,14 +63,14 @@ sudo touch /etc/gitlab/disable-postgresql-upgrade CAUTION: **Warning:** This version is affected by [a bug that results in new LFS objects not being replicated to -Geo secondary nodes](https://gitlab.com/gitlab-org/gitlab/issues/32696). The issue is fixed +Geo secondary nodes](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). The issue is fixed in GitLab 12.1. Please upgrade to GitLab 12.1 or later. ## Updating to GitLab 11.11 CAUTION: **Warning:** This version is affected by [a bug that results in new LFS objects not being replicated to -Geo secondary nodes](https://gitlab.com/gitlab-org/gitlab/issues/32696). The issue is fixed +Geo secondary nodes](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). The issue is fixed in GitLab 12.1. Please upgrade to GitLab 12.1 or later. ## Updating to GitLab 10.8 diff --git a/doc/administration/git_annex.md b/doc/administration/git_annex.md index 0d44ed9312c..c4c38f8e683 100644 --- a/doc/administration/git_annex.md +++ b/doc/administration/git_annex.md @@ -5,7 +5,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/git_annex.html' # Git annex > **Warning:** GitLab has [completely -removed](https://gitlab.com/gitlab-org/gitlab/issues/1648) in GitLab 9.0 (2017/03/22). +removed](https://gitlab.com/gitlab-org/gitlab/-/issues/1648) in GitLab 9.0 (2017/03/22). Read through the [migration guide from git-annex to Git LFS](../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md). The biggest limitation of Git, compared to some older centralized version diff --git a/doc/administration/git_protocol.md b/doc/administration/git_protocol.md index a8e785f9344..e1600d972bd 100644 --- a/doc/administration/git_protocol.md +++ b/doc/administration/git_protocol.md @@ -4,9 +4,9 @@ description: "Set and configure Git protocol v2" # Configuring Git Protocol v2 -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/46555) in GitLab 11.4. -> - [Temporarily disabled](https://gitlab.com/gitlab-org/gitlab-foss/issues/55769) in GitLab 11.5.8, 11.6.6, 11.7.1, and 11.8+. -> - [Re-enabled](https://gitlab.com/gitlab-org/gitlab/issues/27828) in GitLab 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/46555) in GitLab 11.4. +> - [Temporarily disabled](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55769) in GitLab 11.5.8, 11.6.6, 11.7.1, and 11.8+. +> - [Re-enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/27828) in GitLab 12.8. Git protocol v2 improves the v1 wire protocol in several ways and is enabled by default in GitLab for HTTP requests. In order to enable SSH, @@ -24,7 +24,7 @@ From the server side, if we want to configure SSH we need to set the `sshd` server to accept the `GIT_PROTOCOL` environment. In installations using [GitLab Helm Charts](https://docs.gitlab.com/charts/) -and [All-in-one docker image](https://docs.gitlab.com/omnibus/docker/), the SSH +and [All-in-one Docker image](https://docs.gitlab.com/omnibus/docker/), the SSH service is already configured to accept the `GIT_PROTOCOL` environment and users need not do anything more. diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 14b0a6bd450..1469ed64004 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -1,14 +1,25 @@ +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + # Gitaly -[Gitaly](https://gitlab.com/gitlab-org/gitaly) is the service that -provides high-level RPC access to Git repositories. Without it, no other -components can read or write Git data. GitLab components that access Git -repositories (GitLab Rails, GitLab Shell, GitLab Workhorse, etc.) act as clients -to Gitaly. End users do not have direct access to Gitaly. +[Gitaly](https://gitlab.com/gitlab-org/gitaly) is the service that provides high-level RPC access to +Git repositories. Without it, no GitLab components can read or write Git data. + +In the Gitaly documentation: -On this page, *Gitaly server* refers to a standalone node that only runs Gitaly -and *Gitaly client* is a GitLab Rails app node that runs all other processes -except Gitaly. +- **Gitaly server** refers to any node that runs Gitaly itself. +- **Gitaly client** refers to any node that runs a process that makes requests of the + Gitaly server. Processes include, but are not limited to: + - [GitLab Rails application](https://gitlab.com/gitlab-org/gitlab). + - [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell). + - [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse). + +GitLab end users do not have direct access to Gitaly. CAUTION: **Caution:** From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, @@ -17,19 +28,20 @@ support for NFS for Git repositories is scheduled to be removed. Upgrade to ## Architecture -Here's a high-level architecture overview of how Gitaly is used. +The following is a high-level architecture overview of how Gitaly is used.  -## Configuring Gitaly +## Configure Gitaly The Gitaly service itself is configured via a [TOML configuration file](reference.md). -If you want to change any of its settings: +To change Gitaly settings: **For Omnibus GitLab** -1. Edit `/etc/gitlab/gitlab.rb` and add or change the [Gitaly settings](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/1dd07197c7e5ae23626aad5a4a070a800b670380/files/gitlab-config-template/gitlab.rb.template#L1622-1676). +1. Edit `/etc/gitlab/gitlab.rb` and add or change the + [Gitaly settings](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/1dd07197c7e5ae23626aad5a4a070a800b670380/files/gitlab-config-template/gitlab.rb.template#L1622-1676). 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). **For installations from source** @@ -37,92 +49,121 @@ If you want to change any of its settings: 1. Edit `/home/git/gitaly/config.toml` and add or change the [Gitaly settings](https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example). 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -## Running Gitaly on its own server +## Run Gitaly on its own server + +By default, Gitaly is run on the same server as Gitaly clients and is +[configured as above](#configure-gitaly). Single-server installations are best served by +this default configuration used by: -This is an optional way to deploy Gitaly which can benefit GitLab -installations that are larger than a single machine. Most -installations will be better served with the default configuration -used by Omnibus and the GitLab source installation guide. -Follow transition to Gitaly on its own server, [Gitaly servers will need to be upgraded before other servers in your cluster](https://docs.gitlab.com/omnibus/update/#upgrading-gitaly-servers). +- [Omnibus GitLab](https://docs.gitlab.com/omnibus/). +- The GitLab [source installation guide](../../install/installation.md). -Starting with GitLab 11.4, Gitaly is able to serve all Git requests without -requiring a shared NFS mount for Git repository data. -Between 11.4 and 11.8 the exception was the -[Elasticsearch indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). -But since 11.8 the indexer uses Gitaly for data access as well. NFS can still -be leveraged for redundancy on block level of the Git data. But only has to -be mounted on the Gitaly server. +However, Gitaly can be deployed to its own server, which can benefit GitLab installations that span +multiple machines. -From GitLab v11.8 to v12.2, it is possible to use Elasticsearch in conjunction with -a Gitaly setup that isn't utilising NFS. In order to use Elasticsearch in this -scenario, the [new repository indexer](../../integration/elasticsearch.md#elasticsearch-repository-indexer) -needs to be enabled in your GitLab configuration. [Since GitLab v12.3](https://gitlab.com/gitlab-org/gitlab/issues/6481), -the new indexer becomes the default and no configuration is required. +NOTE: **Note:** +When configured to run on their own servers, Gitaly servers +[must be upgraded](https://docs.gitlab.com/omnibus/update/#upgrading-gitaly-servers) before Gitaly +clients in your cluster. + +The process for setting up Gitaly on its own server is: + +1. [Install Gitaly](#install-gitaly). +1. [Configure authentication](#configure-authentication). +1. [Configure Gitaly servers](#configure-gitaly-servers). +1. [Configure Gitaly clients](#configure-gitaly-clients). + +When running Gitaly on its own server, note the following regarding GitLab versions: + +- From GitLab 11.4, Gitaly was able to serve all Git requests without requiring a shared NFS mount + for Git repository data, except for the + [Elasticsearch indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). +- From GitLab 11.8, the Elasticsearch indexer uses Gitaly for data access as well. NFS can still be + leveraged for redundancy on block-level Git data, but only has to be mounted on the Gitaly + servers. +- From GitLab 11.8 to 12.2, it is possible to use Elasticsearch in a Gitaly setup that doesn't use + NFS. In order to use Elasticsearch in these versions, the + [repository indexer](../../integration/elasticsearch.md#elasticsearch-repository-indexer) + must be enabled in your GitLab configuration. +- [Since GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481), the new indexer is + the default and no configuration is required. ### Network architecture -The following list depicts what the network architecture of Gitaly is: +The following list depicts the network architecture of Gitaly: - GitLab Rails shards repositories into [repository storages](../repository_storage_paths.md). -- `/config/gitlab.yml` contains a map from storage names to - `(Gitaly address, Gitaly token)` pairs. -- the `storage name` -\> `(Gitaly address, Gitaly token)` map in - `/config/gitlab.yml` is the single source of truth for the Gitaly network - topology. +- `/config/gitlab.yml` contains a map from storage names to `(Gitaly address, Gitaly token)` pairs. +- The `storage name` -\> `(Gitaly address, Gitaly token)` map in `/config/gitlab.yml` is the single + source of truth for the Gitaly network topology. - A `(Gitaly address, Gitaly token)` corresponds to a Gitaly server. - A Gitaly server hosts one or more storages. -- A GitLab server can use one or more Gitaly servers. -- Gitaly addresses must be specified in such a way that they resolve - correctly for ALL Gitaly clients. -- Gitaly clients are: Puma/Unicorn, Sidekiq, GitLab Workhorse, - GitLab Shell, Elasticsearch Indexer, and Gitaly itself. +- A Gitaly client can use one or more Gitaly servers. +- Gitaly addresses must be specified in such a way that they resolve correctly for **all** Gitaly + clients. +- Gitaly clients are: + - Puma or Unicorn. + - Sidekiq. + - GitLab Workhorse. + - GitLab Shell. + - Elasticsearch indexer. + - Gitaly itself. - A Gitaly server must be able to make RPC calls **to itself** via its own `(Gitaly address, Gitaly token)` pair as specified in `/config/gitlab.yml`. -- Gitaly servers must not be exposed to the public internet as Gitaly's network - traffic is unencrypted by default. The use of firewall is highly recommended - to restrict access to the Gitaly server. Another option is to - [use TLS](#tls-support). -- Authentication is done through a static token which is shared among the Gitaly - and GitLab Rails nodes. +- Authentication is done through a static token which is shared among the Gitaly and GitLab Rails + nodes. + +DANGER: **Danger:** +Gitaly servers must not be exposed to the public internet as Gitaly's network traffic is unencrypted +by default. The use of firewall is highly recommended to restrict access to the Gitaly server. +Another option is to [use TLS](#tls-support). + +In the following sections, we describe how to configure two Gitaly servers with secret token +`abc123secret`: -Below we describe how to configure two Gitaly servers one at -`gitaly1.internal` and the other at `gitaly2.internal` -with secret token `abc123secret`. We assume -your GitLab installation has three repository storages: `default`, -`storage1` and `storage2`. You can use as little as just one server with one -repository storage if desired. +- `gitaly1.internal`. +- `gitaly2.internal`. -Note: **Note:** The token referred to throughout the Gitaly documentation is -just an arbitrary password selected by the administrator. It is unrelated to -tokens created for the GitLab API or other similar web API tokens. +We assume your GitLab installation has three repository storages: -### 1. Installation +- `default`. +- `storage1`. +- `storage2`. -First install Gitaly on each Gitaly server using either -Omnibus GitLab or install it from source: +You can use as few as one server with one repository storage if desired. -- For Omnibus GitLab: [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page but - **_do not_** provide the `EXTERNAL_URL=` value. -- From source: [Install Gitaly](../../install/installation.md#install-gitaly). +NOTE: **Note:** +The token referred to throughout the Gitaly documentation is just an arbitrary password selected by +the administrator. It is unrelated to tokens created for the GitLab API or other similar web API +tokens. + +### Install Gitaly + +Install Gitaly on each Gitaly server using either Omnibus GitLab or install it from source: + +- For Omnibus GitLab, [download and install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want but **do not** provide the `EXTERNAL_URL=` value. +- To install from source, follow the steps at + [Install Gitaly](../../install/installation.md#install-gitaly). -### 2. Authentication +### Configure authentication -Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests -to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. +Gitaly and GitLab use two shared secrets for authentication: + +- One to authenticate gRPC requests to Gitaly. +- A second for authentication callbacks from GitLab Shell to the GitLab internal API. **For Omnibus GitLab** To configure the Gitaly token: -1. On the client server, edit `/etc/gitlab/gitlab.rb`: +1. On the Gitaly clients, edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_rails['gitaly_token'] = 'abc123secret' ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - 1. On the Gitaly server, edit `/etc/gitlab/gitlab.rb`: ```ruby @@ -131,22 +172,24 @@ To configure the Gitaly token: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -There are two ways to configure the GitLab-Shell token: +There are two ways to configure the GitLab Shell token. -1. Copy `/etc/gitlab/gitlab-secrets.json` from the client server to same path on the Gitaly server. -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +Method 1: + +1. Copy `/etc/gitlab/gitlab-secrets.json` from the Gitaly client to same path on the Gitaly servers + (and any other Gitaly clients). +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) on Gitaly servers. -**OR** +Method 2: -1. On the client server, edit `/etc/gitlab/gitlab.rb`: +1. On the Gitaly clients, edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_shell['secret_token'] = 'shellsecret' ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -1. On the Gitaly server, edit `/etc/gitlab/gitlab.rb`: +1. On the Gitaly servers, edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_shell['secret_token'] = 'shellsecret' @@ -156,9 +199,9 @@ There are two ways to configure the GitLab-Shell token: **For installations from source** -1. Copy `/home/git/gitlab/.gitlab_shell_secret` from the client server to the same path on the Gitaly -server. -1. On the client server, edit `/home/git/gitlab/config/gitlab.yml`: +1. Copy `/home/git/gitlab/.gitlab_shell_secret` from the Gitaly client to the same path on the + Gitaly servers (and any other Gitaly clients). +1. On the Gitaly clients, edit `/home/git/gitlab/config/gitlab.yml`: ```yaml gitlab: @@ -167,17 +210,22 @@ server. ``` 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. On the Gitaly servers, edit `/home/git/gitaly/config.toml`: + + ```toml + [auth] + token = 'abc123secret' + ``` -### 3. Gitaly server configuration +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -Next, on the Gitaly servers, you need to configure storage paths, enable -the network listener and configure the token. +### Configure Gitaly servers -NOTE: **Note:** If you want to reduce the risk of downtime when you enable -authentication you can temporarily disable enforcement, see [the -documentation on configuring Gitaly -authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configuration/README.md#authentication) -. +On the Gitaly servers, you must configure storage paths and enable the network listener. + +If you want to reduce the risk of downtime when you enable authentication, you can temporarily +disable enforcement. For more information, see the documentation on configuring +[Gitaly authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configuration/README.md#authentication). **For Omnibus GitLab** @@ -199,6 +247,7 @@ authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configurati sidekiq['enable'] = false gitlab_workhorse['enable'] = false grafana['enable'] = false + gitlab_exporter['enable'] = false # If you run a separate monitoring node you can disable these services alertmanager['enable'] = false @@ -211,7 +260,6 @@ authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configurati # prometheus['monitor_kubernetes'] = false # If you don't want to run monitoring services uncomment the following (not recommended) - # gitlab_exporter['enable'] = false # node_exporter['enable'] = false # Prevent database connections during 'gitlab-ctl reconfigure' @@ -221,7 +269,7 @@ authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configurati # Configure the gitlab-shell API callback URL. Without this, `git push` will # fail. This can be your 'front door' GitLab URL or an internal load # balancer. - # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from Gitaly client to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' # Make Gitaly accept connections on all network interfaces. You must use @@ -230,7 +278,7 @@ authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configurati gitaly['listen_addr'] = "0.0.0.0:8075" ``` -1. Append the following to `/etc/gitlab/gitlab.rb` for each respective server: +1. Append the following to `/etc/gitlab/gitlab.rb` for each respective Gitaly server: <!-- updates to following example must also be made at @@ -262,27 +310,24 @@ authentication](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configurati 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Run `sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml` -to confirm that Gitaly can perform callbacks to the internal API. + to confirm that Gitaly can perform callbacks to the GitLab internal API. **For installations from source** -1. On the client node(s), edit `/home/git/gitaly/config.toml`: +1. Edit `/home/git/gitaly/config.toml`: ```toml listen_addr = '0.0.0.0:8075' internal_socket_dir = '/var/opt/gitlab/gitaly' - [auth] - token = 'abc123secret' - [logging] format = 'json' level = 'info' dir = '/var/log/gitaly' ``` -1. Append the following to `/home/git/gitaly/config.toml` for each respective server: +1. Append the following to `/home/git/gitaly/config.toml` for each respective Gitaly server: On `gitaly1.internal`: @@ -304,34 +349,51 @@ to confirm that Gitaly can perform callbacks to the internal API. path = '/srv/gitlab/git-data/repositories' ``` -1. On each Gitaly server, edit `/home/git/gitlab-shell/config.yml`: +1. Edit `/home/git/gitlab-shell/config.yml`: ```yaml gitlab_url: https://gitlab.example.com ``` -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. Save the files and [restart GitLab](../restart_gitlab.md#installations-from-source). 1. Run `sudo -u git /home/git/gitlab-shell/bin/check -config /home/git/gitlab-shell/config.yml` -to confirm that Gitaly can perform callbacks to the internal API. + to confirm that Gitaly can perform callbacks to the GitLab internal API. + +### Configure Gitaly clients -### 4. Converting clients to use the Gitaly server +As the final step, you must update Gitaly clients to switch from using local Gitaly service to use +the Gitaly servers you just configured. -As the final step, you need to update the client machines to switch from using -their local Gitaly service to the new Gitaly server you just configured. This -is a risky step because if there is any sort of network, firewall, or name -resolution problem preventing your GitLab server from reaching the Gitaly server, -then all Gitaly requests will fail. +This can be risky because anything that prevents your Gitaly clients from reaching the Gitaly +servers will cause all Gitaly requests to fail. For example, any sort of network, firewall, or name +resolution problems. -Additionally, you need to -[disable Rugged if previously manually enabled](../high_availability/nfs.md#improving-nfs-performance-with-gitlab). +Additionally, you must [disable Rugged](../high_availability/nfs.md#improving-nfs-performance-with-gitlab) +if previously enabled manually. -We assume that your `gitaly1.internal` Gitaly server can be reached at -`gitaly1.internal:8075` from your GitLab server, and that Gitaly server -can read and write to `/mnt/gitlab/default` and `/mnt/gitlab/storage1`. +Gitaly makes the following assumptions: -We assume also that your `gitaly2.internal` Gitaly server can be reached at -`gitaly2.internal:8075` from your GitLab server, and that Gitaly server -can read and write to `/mnt/gitlab/storage2`. +- Your `gitaly1.internal` Gitaly server can be reached at `gitaly1.internal:8075` from your Gitaly + clients, and that Gitaly server can read and write to `/mnt/gitlab/default` and + `/mnt/gitlab/storage1`. +- Your `gitaly2.internal` Gitaly server can be reached at `gitaly2.internal:8075` from your Gitaly + clients, and that Gitaly server can read and write to `/mnt/gitlab/storage2`. +- Your `gitaly1.internal` and `gitaly2.internal` Gitaly servers can reach each other. + +Note you can't a use mixed setup, with at least one of your Gitaly servers configured as a local +server with the `path` setting provided. This is because other Gitaly instances can't communicate +with it. The following setup is _incorrect_, because: + +- You must replace `path` with `gitaly_address` containing a proper value. +- The address must be reachable from the other two addresses provided. + +```ruby +git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'path' => '/var/opt/gitlab/git-data' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, +}) +``` **For Omnibus GitLab** @@ -346,7 +408,8 @@ can read and write to `/mnt/gitlab/storage2`. ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the client can connect to Gitaly. +1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the Gitaly client can connect to Gitaly + servers. 1. Tail the logs to see the requests: ```shell @@ -373,39 +436,38 @@ can read and write to `/mnt/gitlab/storage2`. ``` NOTE: **Note:** - `/some/dummy/path` should be set to a local folder that exists, however no - data will be stored in this folder. This will no longer be necessary after - [this issue](https://gitlab.com/gitlab-org/gitaly/issues/1282) is resolved. + `/some/dummy/path` should be set to a local folder that exists, however no data will be stored in + this folder. This will no longer be necessary after + [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -1. Run `sudo -u git -H bundle exec rake gitlab:gitaly:check RAILS_ENV=production` to -confirm the client can connect to Gitaly. +1. Run `sudo -u git -H bundle exec rake gitlab:gitaly:check RAILS_ENV=production` to confirm the + Gitaly client can connect to Gitaly servers. 1. Tail the logs to see the requests: ```shell tail -f /home/git/gitlab/log/gitaly.log ``` -When you tail the Gitaly logs on your Gitaly server you should see requests -coming in. One sure way to trigger a Gitaly request is to clone a repository -from your GitLab server over HTTP. +When you tail the Gitaly logs on your Gitaly server, you should see requests coming in. One sure way +to trigger a Gitaly request is to clone a repository from GitLab over HTTP or HTTPS. DANGER: **Danger:** -If you have [Server hooks](../server_hooks.md) configured, -either per repository or globally, you must move these to the Gitaly node. -If you have multiple Gitaly nodes, copy your server hook(s) to all nodes. +If you have [server hooks](../server_hooks.md) configured, either per repository or globally, you +must move these to the Gitaly servers. If you have multiple Gitaly servers, copy your server hooks +to all Gitaly servers. ### Disabling the Gitaly service in a cluster environment If you are running Gitaly [as a remote -service](#running-gitaly-on-its-own-server) you may want to disable +service](#run-gitaly-on-its-own-server) you may want to disable the local Gitaly service that runs on your GitLab server by default. Disabling Gitaly only makes sense when you run GitLab in a custom cluster configuration, where different services run on different machines. Disabling Gitaly on all machines in the cluster is not a valid configuration. -To disable Gitaly on a client node: +To disable Gitaly on a GitLab server: **For Omnibus GitLab** @@ -436,13 +498,16 @@ with a Gitaly instance that listens for secure connections you will need to use scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. You will need to bring your own certificates as this isn't provided automatically. -The certificate, or its certificate authority, must be installed on all Gitaly -nodes (including the Gitaly node using the certificate) and on all client nodes +The certificate corresponding to each Gitaly server will need to be installed +on that Gitaly server. + +Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers +(including the Gitaly server using the certificate) and on all Gitaly clients that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). NOTE: **Note** -The self-signed certificate must specify the address you use to access the +The certificate must specify the address you use to access the Gitaly server. If you are addressing the Gitaly server by a hostname, you can either use the Common Name field for this, or add it as a Subject Alternative Name. If you are addressing the Gitaly server by its IP address, you must add it @@ -459,7 +524,14 @@ To configure Gitaly with TLS: **For Omnibus GitLab** -1. On the client node(s), edit `/etc/gitlab/gitlab.rb` as follows: +1. Create certificates for Gitaly servers. +1. On the Gitaly clients, copy the certificates, or their certificate authority, into the `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. On the Gitaly clients, edit `git_data_dirs` in `/etc/gitlab/gitlab.rb` as follows: ```ruby git_data_dirs({ @@ -467,19 +539,10 @@ To configure Gitaly with TLS: 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, }) - - gitlab_rails['gitaly_token'] = 'abc123secret' - gitlab_shell['secret_token'] = 'shellsecret' ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) on client node(s). -1. On the client node(s), copy the cert into the `/etc/gitlab/trusted-certs`: - - ```shell - sudo cp cert.pem /etc/gitlab/trusted-certs/ - ``` - -1. On the Gitaly server, create the `/etc/gitlab/ssl` directory and copy your key and certificate there: +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. On the Gitaly servers, create the `/etc/gitlab/ssl` directory and copy your key and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl @@ -488,14 +551,14 @@ To configure Gitaly with TLS: sudo chmod 644 key.pem cert.pem ``` -1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when -calling into itself: +1. Copy all Gitaly server certificates, or their certificate authority, to `/etc/gitlab/trusted-certs` so Gitaly server will trust the certificate when +calling into itself or other Gitaly servers: - ```shell - sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ - ``` + ```shell + sudo cp cert1.pem cert2.pem /etc/gitlab/trusted-certs/ + ``` -1. On the Gitaly server node(s), edit `/etc/gitlab/gitlab.rb` and add: +1. Edit `/etc/gitlab/gitlab.rb` and add: <!-- updates to following example must also be made at @@ -508,23 +571,23 @@ calling into itself: gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) on Gitaly server node(s). +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. (Optional) After [verifying that all Gitaly traffic is being served over TLS](#observe-type-of-gitaly-connections), you can improve security by disabling non-TLS connections by commenting out or deleting `gitaly['listen_addr']` in `/etc/gitlab/gitlab.rb`, saving the file, - and [reconfiguring GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) - on Gitaly server node(s). + and [reconfiguring GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). **For installations from source** -1. On the client node(s), add the cert to the system trusted certs: +1. Create certificates for Gitaly servers. +1. On the Gitaly clients, copy the certificates into the system trusted certificates: ```shell sudo cp cert.pem /usr/local/share/ca-certificates/gitaly.crt sudo update-ca-certificates ``` -1. On the client node(s), edit `/home/git/gitlab/config/gitlab.yml` as follows: +1. On the Gitaly clients, edit `storages` in `/home/git/gitlab/config/gitlab.yml` as follows: ```yaml gitlab: @@ -539,28 +602,21 @@ calling into itself: storage2: gitaly_address: tls://gitaly2.internal:9999 path: /some/dummy/path - - gitaly: - token: 'abc123secret' ``` NOTE: **Note:** `/some/dummy/path` should be set to a local folder that exists, however no data will be stored in this folder. This will no longer be necessary after - [this issue](https://gitlab.com/gitlab-org/gitaly/issues/1282) is resolved. + [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/1282) is resolved. -1. Save the file and[restart GitLab](../restart_gitlab.md#installations-from-source) -on client node(s). -1. Copy `/home/git/gitlab/.gitlab_shell_secret` from the client server to the same -path on the Gitaly server. -1. On the Gitaly server, create or edit `/etc/default/gitlab` and add: +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. On the Gitaly servers, create or edit `/etc/default/gitlab` and add: ```shell export SSL_CERT_DIR=/etc/gitlab/ssl ``` -1. Save the file. -1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: +1. On the Gitaly servers, create the `/etc/gitlab/ssl` directory and copy your key and certificate there: ```shell sudo mkdir -p /etc/gitlab/ssl @@ -569,15 +625,14 @@ path on the Gitaly server. sudo chmod 644 key.pem cert.pem ``` -1. On the Gitaly server, add the cert to the system trusted certs so Gitaly will trust it -when calling into itself: +1. Copy all Gitaly server certificates, or their certificate authority, to the system trusted certificates so Gitaly server will trust the certificate when calling into itself or other Gitaly servers. ```shell sudo cp cert.pem /usr/local/share/ca-certificates/gitaly.crt sudo update-ca-certificates ``` -1. On the Gitaly server node(s), edit `/home/git/gitaly/config.toml` and add: +1. Edit `/home/git/gitaly/config.toml` and add: ```toml tls_listen_addr = '0.0.0.0:9999' @@ -587,12 +642,11 @@ when calling into itself: key_path = '/etc/gitlab/ssl/key.pem' ``` -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) on Gitaly server node(s). +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). 1. (Optional) After [verifying that all Gitaly traffic is being served over TLS](#observe-type-of-gitaly-connections), you can improve security by disabling non-TLS connections by commenting out or deleting `listen_addr` in `/home/git/gitaly/config.toml`, saving the file, - and [restarting GitLab](../restart_gitlab.md#installations-from-source) - on Gitaly server node(s). + and [restarting GitLab](../restart_gitlab.md#installations-from-source). ### Observe type of Gitaly connections @@ -673,8 +727,8 @@ gitaly['concurrency'] = [ This will limit the number of in-flight RPC calls for the given RPC's. The limit is applied per repository. In the example above, each on the -Gitaly server can have at most 20 simultaneous PostUploadPack calls in -flight, and the same for SSHUploadPack. If another request comes in for +Gitaly server can have at most 20 simultaneous `PostUploadPack` calls in +flight, and the same for `SSHUploadPack`. If another request comes in for a repository that has used up its 20 slots, that request will get queued. @@ -686,7 +740,7 @@ reporting about the concurrency limiter. In Prometheus, look for the `gitaly_rate_limiting_seconds` metrics. The name of the Prometheus metric is not quite right because this is a -concurrency limiter, not a rate limiter. If a client makes 1000 requests +concurrency limiter, not a rate limiter. If a Gitaly client makes 1000 requests in a row in a very short timespan, the concurrency will not exceed 1, and this mechanism (the concurrency limiter) will do nothing. @@ -697,7 +751,7 @@ downtime, or causes outages, or both. If you are careful, though, you *can* rotate Gitaly credentials without a service interruption. This procedure also works if you are running GitLab on a single server. -In that case, "Gitaly servers" and "Gitaly clients" refers to the same +In that case, "Gitaly server" and "Gitaly client" refers to the same machine. ### 1. Monitor current authentication behavior @@ -723,7 +777,7 @@ The only non-zero number should have `enforced="true",status="ok"`. If you have other non-zero numbers, something is wrong in your configuration. -The 'status="ok"' number reflects your current request rate. In the example +The `status="ok"` number reflects your current request rate. In the example above, Gitaly is handling about 4000 requests per second. Now you have established that you can monitor the Gitaly authentication @@ -894,9 +948,9 @@ which is still under development as of December 2019. ## Troubleshooting Gitaly -### Checking versions when using standalone Gitaly nodes +### Checking versions when using standalone Gitaly servers -When using standalone Gitaly nodes, you must make sure they are the same version +When using standalone Gitaly servers, you must make sure they are the same version as GitLab to ensure full compatibility. Check **Admin Area > Gitaly Servers** on your GitLab instance and confirm all Gitaly Servers are `Up to date`. @@ -931,8 +985,8 @@ gitaly-debug -h remote: GitLab: 401 Unauthorized ``` -You will need to sync your `gitlab-secrets.json` file with your GitLab -app nodes. +You will need to sync your `gitlab-secrets.json` file with your Gitaly clients (GitLab +app nodes). ### Client side gRPC logs @@ -975,11 +1029,11 @@ If you're running Gitaly on its own server and notice that users can successfully clone and fetch repositories (via both SSH and HTTPS), but can't push to them or make changes to the repository in the web UI without getting a `401 Unauthorized` message, then it's possible Gitaly is failing to authenticate -with the other nodes due to having the [wrong secrets file](#3-gitaly-server-configuration). +with the Gitaly client due to having the [wrong secrets file](#configure-gitaly-servers). Confirm the following are all true: -- When any user performs a `git push` to any repository on this Gitaly node, it +- When any user performs a `git push` to any repository on this Gitaly server, it fails with the following error (note the `401 Unauthorized`): ```shell @@ -993,7 +1047,8 @@ Confirm the following are all true: UI, it immediately fails with a red `401 Unauthorized` banner. - Creating a new project and [initializing it with a README](../../gitlab-basics/create-project.md#blank-projects) successfully creates the project but doesn't create the README. -- When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.html#tail-logs-in-a-console-on-the-server) on an app node and reproducing the error, you get `401` errors +- When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.html#tail-logs-in-a-console-on-the-server) + on a Gitaly client and reproducing the error, you get `401` errors when reaching the `/api/v4/internal/allowed` endpoint: ```shell @@ -1055,14 +1110,16 @@ Confirm the following are all true: [IP] - - [18/Jul/2019:00:30:14 +0000] "POST /api/v4/internal/allowed HTTP/1.1" 401 30 "" "Ruby" ``` -To fix this problem, confirm that your [`gitlab-secrets.json` file](#3-gitaly-server-configuration) -on the Gitaly node matches the one on all other nodes. If it doesn't match, -update the secrets file on the Gitaly node to match the others, then -[reconfigure the node](../restart_gitlab.md#omnibus-gitlab-reconfigure). +To fix this problem, confirm that your [`gitlab-secrets.json` file](#configure-gitaly-servers) +on the Gitaly server matches the one on Gitaly client. If it doesn't match, +update the secrets file on the Gitaly server to match the Gitaly client, then +[reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Command line tools cannot connect to Gitaly -If you are having trouble connecting to a Gitaly node with command line (CLI) tools, and certain actions result in a `14: Connect Failed` error message, it means that gRPC cannot reach your Gitaly node. +If you are having trouble connecting to a Gitaly server with command line (CLI) tools, +and certain actions result in a `14: Connect Failed` error message, +it means that gRPC cannot reach your Gitaly server. Verify that you can reach Gitaly via TCP: @@ -1070,18 +1127,22 @@ Verify that you can reach Gitaly via TCP: sudo gitlab-rake gitlab:tcp_check[GITALY_SERVER_IP,GITALY_LISTEN_PORT] ``` -If the TCP connection fails, check your network settings and your firewall rules. If the TCP connection succeeds, your networking and firewall rules are correct. +If the TCP connection fails, check your network settings and your firewall rules. +If the TCP connection succeeds, your networking and firewall rules are correct. -If you use proxy servers in your command line environment, such as Bash, these can interfere with your gRPC traffic. +If you use proxy servers in your command line environment, such as Bash, these +can interfere with your gRPC traffic. -If you use Bash or a compatible command line environment, run the following commands to determine whether you have proxy servers configured: +If you use Bash or a compatible command line environment, run the following commands +to determine whether you have proxy servers configured: ```shell echo $http_proxy echo $https_proxy ``` -If either of these variables have a value, your Gitaly CLI connections may be getting routed through a proxy which cannot connect to Gitaly. +If either of these variables have a value, your Gitaly CLI connections may be +getting routed through a proxy which cannot connect to Gitaly. To remove the proxy setting, run the following commands (depending on which variables had values): @@ -1092,20 +1153,21 @@ unset https_proxy ### Gitaly not listening on new address after reconfiguring -When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. +When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` +values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. -When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/issues/2521) is resolved. +When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/2521) is resolved. -### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly node +### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly server If this error occurs even though file permissions are correct, it's likely that -the Gitaly node is experiencing +the Gitaly server is experiencing [clock drift](https://en.wikipedia.org/wiki/Clock_drift). -Please ensure that the GitLab and Gitaly nodes are synchronized and use an NTP time +Please ensure that the Gitaly clients and servers are synchronized and use an NTP time server to keep them synchronized if possible. ### Praefect -Praefect is an experimental daemon that allows for replication of the Git data. -It can be setup with omnibus, [as explained here](./praefect.md). +Praefect is a router and transaction manager for Gitaly, and a required +component for running a Gitaly Cluster. For more information see [Gitaly Cluster](praefect.md). diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 0ea83f0e090..3d4e606205e 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -89,7 +92,7 @@ to install GitLab](https://about.gitlab.com/install/). Provision a PostgreSQL server (PostgreSQL 11 or newer). Configuration through the Omnibus GitLab distribution is not yet supported. Follow this -[issue](https://gitlab.com/gitlab-org/gitaly/issues/2476) for updates. +[issue](https://gitlab.com/gitlab-org/gitaly/-/issues/2476) for updates. Prepare all your new nodes by [installing GitLab](https://about.gitlab.com/install/). @@ -319,37 +322,6 @@ application server, or a Gitaly node. } ``` -1. Enable the database replication queue: - - ```ruby - praefect['postgres_queue_enabled'] = true - ``` - - In the next release, database replication queue will be enabled by default. - See [issue #2615](https://gitlab.com/gitlab-org/gitaly/-/issues/2615). - -1. Enable automatic failover by editing `/etc/gitlab/gitlab.rb`: - - ```ruby - praefect['failover_enabled'] = true - praefect['failover_election_strategy'] = 'sql' - ``` - - When automatic failover is enabled, Praefect checks the health of internal - Gitaly nodes. If the primary has a certain amount of health checks fail, it - will promote one of the secondaries to be primary, and demote the primary to - be a secondary. - - NOTE: **Note:** Database leader election will be [enabled by default in the - future](https://gitlab.com/gitlab-org/gitaly/-/issues/2682). - - Caution, **automatic failover** favors availability over consistency and will - cause data loss if changes have not been replicated to the newly elected - primary. In the next release, leader election will [prefer to promote up to - date replicas](https://gitlab.com/gitlab-org/gitaly/-/issues/2642), and it - will be an option to favor consistency by marking [out-of-date repositories - read-only](https://gitlab.com/gitlab-org/gitaly/-/issues/2630). - 1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): @@ -409,7 +381,7 @@ Particular attention should be shown to: `gitaly-2`, and `gitaly-3` as Gitaly storage names. For more information on Gitaly server configuration, see our [Gitaly -documentation](index.md#3-gitaly-server-configuration). +documentation](index.md#configure-gitaly-servers). 1. SSH into the **Gitaly** node and login as root: @@ -424,16 +396,18 @@ documentation](index.md#3-gitaly-server-configuration). postgresql['enable'] = false redis['enable'] = false nginx['enable'] = false - prometheus['enable'] = false grafana['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false prometheus_monitoring['enable'] = false - # Enable only the Praefect service + # Enable only the Gitaly service gitaly['enable'] = true + # Enable Prometheus if needed + prometheus['enable'] = true + # Prevent database connections during 'gitlab-ctl reconfigure' gitlab_rails['rake_cache_clear'] = false gitlab_rails['auto_migrate'] = false @@ -682,8 +656,8 @@ Particular attention should be shown to: Repository > Repository storage** to make the newly configured Praefect cluster the storage location for new Git repositories. - - Deselect the **default** storage location - - Select the **praefect** storage location + - The default option is unchecked. + - The Praefect option is checked.  @@ -744,7 +718,7 @@ Praefect regularly checks the health of each backend Gitaly node. This information can be used to automatically failover to a new primary node if the current primary node is found to be unhealthy. -- **PostgreSQL (recommended):** Enabled by setting +- **PostgreSQL (recommended):** Enabled by default, and equivalent to: `praefect['failover_election_strategy'] = sql`. This configuration option will allow multiple Praefect nodes to coordinate via the PostgreSQL database to elect a primary Gitaly node. This configuration @@ -755,24 +729,21 @@ current primary node is found to be unhealthy. reconfigured in `/etc/gitlab/gitlab.rb` on the Praefect node. Modify the `praefect['virtual_storages']` field by moving the `primary = true` to promote a different Gitaly node to primary. In the steps above, `gitaly-1` was set to - the primary. -- **Memory:** Enabled by setting `praefect['failover_enabled'] = true` in - `/etc/gitlab/gitlab.rb` on the Praefect node. If a sufficient number of health + the primary. Requires `praefect['failover_enabled'] = false` in the configuration. +- **Memory:** Enabled by setting `praefect['failover_election_strategy'] = 'local'` + in `/etc/gitlab/gitlab.rb` on the Praefect node. If a sufficient number of health checks fail for the current primary backend Gitaly node, and new primary will be elected. **Do not use with multiple Praefect nodes!** Using with multiple Praefect nodes is likely to result in a split brain. -NOTE: **Note:**: Praefect does not yet account for replication lag on -the secondaries during the election process, so data loss can occur -during a failover. Follow issue -[#2642](https://gitlab.com/gitlab-org/gitaly/-/issues/2642) for updates. - It is likely that we will implement support for Consul, and a cloud native strategy in the future. ## Identifying Impact of a Primary Node Failure -When a primary Gitaly node fails, there is a chance of data loss. Data loss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The Praefect `dataloss` sub-command helps identify these cases by counting the number of dead replication jobs for each repository within a given time frame. +When a primary Gitaly node fails, there is a chance of data loss. Data loss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The `dataloss` Praefect sub-command helps identify these cases by counting the number of dead replication jobs for each repository. This command must be executed on a Praefect node. + +A time frame to search can be specified with `-from` and `-to`: ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss -from <rfc3339-time> -to <rfc3339-time> @@ -784,9 +755,7 @@ If the time frame is not specified, dead replication jobs from the last six hour sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss Failed replication jobs between [2020-01-02 00:00:00 +0000 UTC, 2020-01-02 06:00:00 +0000 UTC): -example/repository-1: 1 jobs -example/repository-2: 4 jobs -example/repository-3: 2 jobs +@hashed/fa/53/fa539965395b8382145f8370b34eab249cf610d2d6f2943c95b9b9d08a63d4a3.git: 2 jobs ``` To specify a time frame in UTC, run: @@ -797,7 +766,8 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t ### Checking repository checksums -To check a project's checksums across all nodes, the Praefect replicas Rake task can be used: +To check a project's repository checksums across on all Gitaly nodes, the +replicas Rake task can be run on the main GitLab node: ```shell sudo gitlab-rake "gitlab:praefect:replicas[project_id]" diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index e718d8953ca..52fd6fa6900 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -12,7 +15,7 @@ The configuration file is passed as an argument to the `gitaly` executable. This is usually done by either Omnibus GitLab or your [init](https://en.wikipedia.org/wiki/Init) script. -An [example config file](https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example) +An [example configuration file](https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example) can be found in the Gitaly project. ## Format @@ -157,7 +160,7 @@ sum(rate(gitaly_catfile_cache_total{type="hit"}[5m])) / sum(rate(gitaly_catfile_ A Gitaly process uses one or more `gitaly-ruby` helper processes to execute RPC's implemented in Ruby instead of Go. The `[gitaly-ruby]` -section of the config file contains settings for these helper processes. +section of the configuration file contains settings for these helper processes. These processes are known to occasionally suffer from memory leaks. Gitaly restarts its `gitaly-ruby` helpers when their memory exceeds the @@ -190,7 +193,7 @@ For historical reasons the Git hooks that allow GitLab to validate and react to Git pushes. Because Gitaly "owns" Git pushes, GitLab Shell must therefore be installed alongside Gitaly. This will be [simplified in the -future](https://gitlab.com/gitlab-org/gitaly/issues/1226). +future](https://gitlab.com/gitlab-org/gitaly/-/issues/1226). | Name | Type | Required | Description | | ---- | ---- | -------- | ----------- | diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md index 1f22c46a0ad..a87c1f1027f 100644 --- a/doc/administration/high_availability/consul.md +++ b/doc/administration/high_availability/consul.md @@ -7,7 +7,7 @@ type: reference As part of its High Availability stack, GitLab Premium includes a bundled version of [Consul](https://www.consul.io/) that can be managed through `/etc/gitlab/gitlab.rb`. Consul is a service networking solution. When it comes to [GitLab Architecture](../../development/architecture.md), Consul utilization is supported for configuring: 1. [Monitoring in Scaled and Highly Available environments](monitoring_node.md) -1. [PostgreSQL High Availability with Omnibus](database.md#high-availability-with-omnibus-gitlab-premium-only) +1. [PostgreSQL High Availability with Omnibus](../postgresql/replication_and_failover.md) A Consul cluster consists of multiple server agents, as well as client agents that run on other nodes which need to talk to the Consul cluster. @@ -27,7 +27,7 @@ When installing the GitLab package, do not supply `EXTERNAL_URL` value. On each Consul node perform the following: -1. Make sure you collect [`CONSUL_SERVER_NODES`](database.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step, before executing the next step. +1. Make sure you collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step, before executing the next step. 1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index 6f1873af993..75183436046 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -4,1100 +4,17 @@ type: reference # Configuring PostgreSQL for Scaling and High Availability -In this section, you'll be guided through configuring a PostgreSQL database -to be used with GitLab in a highly available environment. +In this section, you'll be guided through configuring a PostgreSQL database to +be used with GitLab in one of our [Scalable and Highly Available Setups](../reference_architectures/index.md). ## Provide your own PostgreSQL instance **(CORE ONLY)** -If you're hosting GitLab on a cloud provider, you can optionally use a -managed service for PostgreSQL. For example, AWS offers a managed Relational -Database Service (RDS) that runs PostgreSQL. +This content has been moved to a [new location](../postgresql/external.md). -If you use a cloud-managed service, or provide your own PostgreSQL: +## Standalone PostgreSQL using Omnibus GitLab **(CORE ONLY)** -1. Set up PostgreSQL according to the - [database requirements document](../../install/requirements.md#database). -1. Set up a `gitlab` username with a password of your choice. The `gitlab` user - needs privileges to create the `gitlabhq_production` database. -1. Configure the GitLab application servers with the appropriate details. - This step is covered in [Configuring GitLab for HA](gitlab.md). +This content has been moved to a [new location](../postgresql/standalone.md). -## PostgreSQL in a Scaled and Highly Available Environment +## PostgreSQL replication and failover with Omnibus GitLab **(PREMIUM ONLY)** -This section is relevant for [Scalable and Highly Available Setups](../reference_architectures/index.md). - -### Provide your own PostgreSQL instance **(CORE ONLY)** - -If you want to use your own deployed PostgreSQL instance(s), -see [Provide your own PostgreSQL instance](#provide-your-own-postgresql-instance-core-only) -for more details. However, you can use the Omnibus GitLab package to easily -deploy the bundled PostgreSQL. - -### Standalone PostgreSQL using Omnibus GitLab **(CORE ONLY)** - -1. SSH into the PostgreSQL server. -1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab - package you want using **steps 1 and 2** from the GitLab downloads page. - - Do not complete any other steps on the download page. -1. Generate a password hash for PostgreSQL. This assumes you will use the default - username of `gitlab` (recommended). The command will request a password - and confirmation. Use the value that is output by this command in the next - step as the value of `POSTGRESQL_PASSWORD_HASH`. - - ```shell - sudo gitlab-ctl pg-password-md5 gitlab - ``` - -1. Edit `/etc/gitlab/gitlab.rb` and add the contents below, updating placeholder - values appropriately. - - - `POSTGRESQL_PASSWORD_HASH` - The value output from the previous step - - `APPLICATION_SERVER_IP_BLOCKS` - A space delimited list of IP subnets or IP - addresses of the GitLab application servers that will connect to the - database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` - - ```ruby - # Disable all components except PostgreSQL - roles ['postgres_role'] - repmgr['enable'] = false - consul['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - redis_exporter['enable'] = false - gitlab_exporter['enable'] = false - - postgresql['listen_address'] = '0.0.0.0' - postgresql['port'] = 5432 - - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - # ???? - postgresql['trust_auth_cidr_addresses'] = %w(APPLICATION_SERVER_IP_BLOCKS) - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - ``` - - NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Note the PostgreSQL node's IP address or hostname, port, and - plain text password. These will be necessary when configuring the GitLab - application servers later. -1. [Enable monitoring](#enable-monitoring) - -Advanced configuration options are supported and can be added if -needed. - -### High Availability with Omnibus GitLab **(PREMIUM ONLY)** - -> Important notes: -> -> - This document will focus only on configuration supported with [GitLab Premium](https://about.gitlab.com/pricing/), using the Omnibus GitLab package. -> - If you are a Community Edition or Starter user, consider using a cloud hosted solution. -> - This document will not cover installations from source. -> -> - If HA setup is not what you were looking for, see the [database configuration document](https://docs.gitlab.com/omnibus/settings/database.html) -> for the Omnibus GitLab packages. -> -> Please read this document fully before attempting to configure PostgreSQL HA -> for GitLab. -> -> This configuration is GA in EE 10.2. - -The recommended configuration for a PostgreSQL HA requires: - -- A minimum of three database nodes - - Each node will run the following services: - - `PostgreSQL` - The database itself - - `repmgrd` - A service to monitor, and handle failover in case of a failure - - `Consul` agent - Used for service discovery, to alert other nodes when failover occurs -- A minimum of three `Consul` server nodes -- A minimum of one `pgbouncer` service node, but it's recommended to have one per database node - - An internal load balancer (TCP) is required when there is more than one `pgbouncer` service node - -You also need to take into consideration the underlying network topology, -making sure you have redundant connectivity between all Database and GitLab instances, -otherwise the networks will become a single point of failure. - -#### Architecture - - - -Database nodes run two services with PostgreSQL: - -- Repmgrd. Monitors the cluster and handles failover when issues with the master occur. The failover consists of: - - Selecting a new master for the cluster. - - Promoting the new node to master. - - Instructing remaining servers to follow the new master node. - - On failure, the old master node is automatically evicted from the cluster, and should be rejoined manually once recovered. -- Consul. Monitors the status of each node in the database cluster and tracks its health in a service definition on the Consul cluster. - -Alongside each PgBouncer, there is a Consul agent that watches the status of the PostgreSQL service. If that status changes, Consul runs a script which updates the configuration and reloads PgBouncer - -##### Connection flow - -Each service in the package comes with a set of [default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#ports). You may need to make specific firewall rules for the connections listed below: - -- Application servers connect to either PgBouncer directly via its [default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#pgbouncer) or via a configured Internal Load Balancer (TCP) that serves multiple PgBouncers. -- PgBouncer connects to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) -- Repmgr connects to the database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) -- PostgreSQL secondaries connect to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) -- Consul servers and agents connect to each others [Consul default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#consul) - -#### Required information - -Before proceeding with configuration, you will need to collect all the necessary -information. - -##### Network information - -PostgreSQL does not listen on any network interface by default. It needs to know -which IP address to listen on in order to be accessible to other services. -Similarly, PostgreSQL access is controlled based on the network source. - -This is why you will need: - -- IP address of each nodes network interface. This can be set to `0.0.0.0` to - listen on all interfaces. It cannot be set to the loopback address `127.0.0.1`. -- Network Address. This can be in subnet (i.e. `192.168.0.0/255.255.255.0`) - or CIDR (i.e. `192.168.0.0/24`) form. - -##### User information - -Various services require different configuration to secure -the communication as well as information required for running the service. -Bellow you will find details on each service and the minimum required -information you need to provide. - -##### Consul information - -When using default setup, minimum configuration requires: - -- `CONSUL_USERNAME`. Defaults to `gitlab-consul` -- `CONSUL_DATABASE_PASSWORD`. Password for the database user. -- `CONSUL_PASSWORD_HASH`. This is a hash generated out of Consul username/password pair. - Can be generated with: - - ```shell - sudo gitlab-ctl pg-password-md5 CONSUL_USERNAME - ``` - -- `CONSUL_SERVER_NODES`. The IP addresses or DNS records of the Consul server nodes. - -Few notes on the service itself: - -- The service runs under a system account, by default `gitlab-consul`. - - If you are using a different username, you will have to specify it. We - will refer to it with `CONSUL_USERNAME`, -- There will be a database user created with read only access to the repmgr - database -- Passwords will be stored in the following locations: - - `/etc/gitlab/gitlab.rb`: hashed - - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed - - `/var/opt/gitlab/consul/.pgpass`: plaintext - -##### PostgreSQL information - -When configuring PostgreSQL, we will set `max_wal_senders` to one more than -the number of database nodes in the cluster. -This is used to prevent replication from using up all of the -available database connections. - -In this document we are assuming 3 database nodes, which makes this configuration: - -```ruby -postgresql['max_wal_senders'] = 4 -``` - -As previously mentioned, you'll have to prepare the network subnets that will -be allowed to authenticate with the database. -You'll also need to supply the IP addresses or DNS records of Consul -server nodes. - -We will need the following password information for the application's database user: - -- `POSTGRESQL_USERNAME`. Defaults to `gitlab` -- `POSTGRESQL_USER_PASSWORD`. The password for the database user -- `POSTGRESQL_PASSWORD_HASH`. This is a hash generated out of the username/password pair. - Can be generated with: - - ```shell - sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME - ``` - -##### PgBouncer information - -When using default setup, minimum configuration requires: - -- `PGBOUNCER_USERNAME`. Defaults to `pgbouncer` -- `PGBOUNCER_PASSWORD`. This is a password for PgBouncer service. -- `PGBOUNCER_PASSWORD_HASH`. This is a hash generated out of PgBouncer username/password pair. - Can be generated with: - - ```shell - sudo gitlab-ctl pg-password-md5 PGBOUNCER_USERNAME - ``` - -- `PGBOUNCER_NODE`, is the IP address or a FQDN of the node running PgBouncer. - -Few notes on the service itself: - -- The service runs as the same system account as the database - - In the package, this is by default `gitlab-psql` -- If you use a non-default user account for PgBouncer service (by default `pgbouncer`), you will have to specify this username. We will refer to this requirement with `PGBOUNCER_USERNAME`. -- The service will have a regular database user account generated for it - - This defaults to `repmgr` -- Passwords will be stored in the following locations: - - `/etc/gitlab/gitlab.rb`: hashed, and in plain text - - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed - -##### Repmgr information - -When using default setup, you will only have to prepare the network subnets that will -be allowed to authenticate with the service. - -Few notes on the service itself: - -- The service runs under the same system account as the database - - In the package, this is by default `gitlab-psql` -- The service will have a superuser database user account generated for it - - This defaults to `gitlab_repmgr` - -#### Installing Omnibus GitLab - -First, make sure to [download/install](https://about.gitlab.com/install/) -Omnibus GitLab **on each node**. - -Make sure you install the necessary dependencies from step 1, -add GitLab package repository from step 2. -When installing the GitLab package, do not supply `EXTERNAL_URL` value. - -#### Configuring the Database nodes - -1. Make sure to [configure the Consul nodes](consul.md). -1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information), [`POSTGRESQL_PASSWORD_HASH`](#postgresql-information), the [number of db nodes](#postgresql-information), and the [network address](#network-information) before executing the next step. - -1. On the master database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - - ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the Consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - # Replace X with value of number of db nodes + 1 - postgresql['max_wal_senders'] = X - postgresql['max_replication_slots'] = X - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) - - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - # - # END user configuration - ``` - - > `postgres_role` was introduced with GitLab 10.3 - -1. On secondary nodes, add all the configuration specified above for primary node - to `/etc/gitlab/gitlab.rb`. In addition, append the following configuration - to inform `gitlab-ctl` that they are standby nodes initially and it need not - attempt to register them as primary node - - ```ruby - # HA setting to specify if a node should attempt to be master on initialization - repmgr['master_on_initialization'] = false - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. [Enable Monitoring](#enable-monitoring) - -> Please note: -> -> - If you want your database to listen on a specific interface, change the config: -> `postgresql['listen_address'] = '0.0.0.0'`. -> - If your PgBouncer service runs under a different user account, -> you also need to specify: `postgresql['pgbouncer_user'] = PGBOUNCER_USERNAME` in -> your configuration. - -##### Database nodes post-configuration - -###### Primary node - -Select one node as a primary node. - -1. Open a database prompt: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - -1. Enable the `pg_trgm` extension: - - ```shell - CREATE EXTENSION pg_trgm; - ``` - -1. Exit the database prompt by typing `\q` and Enter. - -1. Verify the cluster is initialized with one node: - - ```shell - gitlab-ctl repmgr cluster show - ``` - - The output should be similar to the following: - - ```plaintext - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will - refer to the hostname in the next section as `MASTER_NODE_NAME`. If the value - is not an IP address, it will need to be a resolvable name (via DNS or - `/etc/hosts`) - -###### Secondary nodes - -1. Set up the repmgr standby: - - ```shell - gitlab-ctl repmgr standby setup MASTER_NODE_NAME - ``` - - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - # gitlab-ctl repmgr standby setup MASTER_NODE_NAME - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s - ``` - -1. Verify the node now appears in the cluster: - - ```shell - gitlab-ctl repmgr cluster show - ``` - - The output should be similar to the following: - - ```plaintext - Role | Name | Upstream | Connection String - ----------+---------|-----------|------------------------------------------------ - * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -Repeat the above steps on all secondary nodes. - -##### Database checkpoint - -Before moving on, make sure the databases are configured correctly. Run the -following command on the **primary** node to verify that replication is working -properly: - -```shell -gitlab-ctl repmgr cluster show -``` - -The output should be similar to: - -```plaintext -Role | Name | Upstream | Connection String -----------+--------------|--------------|-------------------------------------------------------------------- -* master | MASTER | | host=MASTER port=5432 user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host=STANDBY port=5432 user=gitlab_repmgr dbname=gitlab_repmgr -``` - -If the 'Role' column for any node says "FAILED", check the -[Troubleshooting section](#troubleshooting) before proceeding. - -Also, check that the check master command works successfully on each node: - -```shell -su - gitlab-consul -gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' -``` - -This command relies on exit codes to tell Consul whether a particular node is a master -or secondary. The most important thing here is that this command does not produce errors. -If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. -Check the [Troubleshooting section](#troubleshooting) before proceeding. - -#### Configuring the PgBouncer node - -See our [documentation for PgBouncer](pgbouncer.md) for information on running PgBouncer as part of an HA setup. - -#### Configuring the Application nodes - -These will be the nodes running the `gitlab-rails` service. You may have other -attributes set, but the following need to be set. - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - # Disable PostgreSQL on the application node - postgresql['enable'] = false - - gitlab_rails['db_host'] = 'PGBOUNCER_NODE' or 'INTERNAL_LOAD_BALANCER' - gitlab_rails['db_port'] = 6432 - gitlab_rails['db_password'] = 'POSTGRESQL_USER_PASSWORD' - gitlab_rails['auto_migrate'] = false - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -##### Application node post-configuration - -Ensure that all migrations ran: - -```shell -gitlab-rake gitlab:db:configure -``` - -> **Note**: If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to -PostgreSQL it may be that your PgBouncer node's IP address is missing from -PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See -[PgBouncer error `ERROR: pgbouncer cannot connect to server`](#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) -in the Troubleshooting section before proceeding. - -##### Ensure GitLab is running - -At this point, your GitLab instance should be up and running. Verify you are -able to login, and create issues and merge requests. If you have troubles check -the [Troubleshooting section](#troubleshooting). - -#### Example configuration - -Here we'll show you some fully expanded example configurations. - -##### Example recommended setup - -This example uses 3 Consul servers, 3 PgBouncer servers (with associated internal load balancer), -3 PostgreSQL servers, and 1 application node. - -We start with all servers on the same 10.6.0.0/16 private network range, they -can connect to each freely other on those addresses. - -Here is a list and description of each machine and the assigned IP: - -- `10.6.0.11`: Consul 1 -- `10.6.0.12`: Consul 2 -- `10.6.0.13`: Consul 3 -- `10.6.0.20`: Internal Load Balancer -- `10.6.0.21`: PgBouncer 1 -- `10.6.0.22`: PgBouncer 2 -- `10.6.0.23`: PgBouncer 3 -- `10.6.0.31`: PostgreSQL master -- `10.6.0.32`: PostgreSQL secondary -- `10.6.0.33`: PostgreSQL secondary -- `10.6.0.41`: GitLab application - -All passwords are set to `toomanysecrets`, please do not use this password or derived hashes and the `external_url` for GitLab is `http://gitlab.example.com`. - -Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back. - -##### Example recommended setup for Consul servers - -On each server edit `/etc/gitlab/gitlab.rb`: - -```ruby -# Disable all components except Consul -roles ['consul_role'] - -consul['configuration'] = { - server: true, - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) -} -consul['monitoring_service_discovery'] = true -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -##### Example recommended setup for PgBouncer servers - -On each server edit `/etc/gitlab/gitlab.rb`: - -```ruby -# Disable all components except Pgbouncer and Consul agent -roles ['pgbouncer_role'] - -# Configure PgBouncer -pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - -pgbouncer['users'] = { - 'gitlab-consul': { - password: '5e0e3263571e3704ad655076301d6ebe' - }, - 'pgbouncer': { - password: '771a8625958a529132abe6f1a4acb19c' - } -} - -consul['watchers'] = %w(postgresql) -consul['enable'] = true -consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) -} -consul['monitoring_service_discovery'] = true -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -##### Internal load balancer setup - -An internal load balancer (TCP) is then required to be setup to serve each PgBouncer node (in this example on the IP of `10.6.0.20`). An example of how to do this can be found in the [PgBouncer Configure Internal Load Balancer](pgbouncer.md#configure-the-internal-load-balancer) section. - -##### Example recommended setup for PostgreSQL servers - -###### Primary node - -On primary node edit `/etc/gitlab/gitlab.rb`: - -```ruby -# Disable all components except PostgreSQL and Repmgr and Consul -roles ['postgres_role'] - -# PostgreSQL configuration -postgresql['listen_address'] = '0.0.0.0' -postgresql['hot_standby'] = 'on' -postgresql['wal_level'] = 'replica' -postgresql['shared_preload_libraries'] = 'repmgr_funcs' - -# Disable automatic database migrations -gitlab_rails['auto_migrate'] = false - -postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' -postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' -postgresql['max_wal_senders'] = 4 - -postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) -repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) - -# Configure the Consul agent -consul['services'] = %w(postgresql) -consul['enable'] = true -consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) -} -consul['monitoring_service_discovery'] = true -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -###### Secondary nodes - -On secondary nodes, edit `/etc/gitlab/gitlab.rb` and add all the configuration -added to primary node, noted above. In addition, append the following -configuration: - -```ruby -# HA setting to specify if a node should attempt to be master on initialization -repmgr['master_on_initialization'] = false -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -##### Example recommended setup for application server - -On the server edit `/etc/gitlab/gitlab.rb`: - -```ruby -external_url 'http://gitlab.example.com' - -gitlab_rails['db_host'] = '10.6.0.20' # Internal Load Balancer for PgBouncer nodes -gitlab_rails['db_port'] = 6432 -gitlab_rails['db_password'] = 'toomanysecrets' -gitlab_rails['auto_migrate'] = false - -postgresql['enable'] = false -pgbouncer['enable'] = false -consul['enable'] = true - -# Configure Consul agent -consul['watchers'] = %w(postgresql) - -pgbouncer['users'] = { - 'gitlab-consul': { - password: '5e0e3263571e3704ad655076301d6ebe' - }, - 'pgbouncer': { - password: '771a8625958a529132abe6f1a4acb19c' - } -} - -consul['configuration'] = { - retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) -} -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -##### Example recommended setup manual steps - -After deploying the configuration follow these steps: - -1. On `10.6.0.31`, our primary database - - Enable the `pg_trgm` extension - - ```shell - gitlab-psql -d gitlabhq_production - ``` - - ```shell - CREATE EXTENSION pg_trgm; - ``` - -1. On `10.6.0.32`, our first standby database - - Make this node a standby of the primary - - ```shell - gitlab-ctl repmgr standby setup 10.6.0.21 - ``` - -1. On `10.6.0.33`, our second standby database - - Make this node a standby of the primary - - ```shell - gitlab-ctl repmgr standby setup 10.6.0.21 - ``` - -1. On `10.6.0.41`, our application server - - Set `gitlab-consul` user's PgBouncer password to `toomanysecrets` - - ```shell - gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul - ``` - - Run database migrations - - ```shell - gitlab-rake gitlab:db:configure - ``` - -#### Example minimal setup - -This example uses 3 PostgreSQL servers, and 1 application node (with PgBouncer setup alongside). - -It differs from the [recommended setup](#example-recommended-setup) by moving the Consul servers into the same servers we use for PostgreSQL. -The trade-off is between reducing server counts, against the increased operational complexity of needing to deal with PostgreSQL [failover](#failover-procedure) and [restore](#restore-procedure) procedures in addition to [Consul outage recovery](consul.md#outage-recovery) on the same set of machines. - -In this example we start with all servers on the same 10.6.0.0/16 private network range, they can connect to each freely other on those addresses. - -Here is a list and description of each machine and the assigned IP: - -- `10.6.0.21`: PostgreSQL master -- `10.6.0.22`: PostgreSQL secondary -- `10.6.0.23`: PostgreSQL secondary -- `10.6.0.31`: GitLab application - -All passwords are set to `toomanysecrets`, please do not use this password or derived hashes. - -The `external_url` for GitLab is `http://gitlab.example.com` - -Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back. - -##### Example minimal configuration for database servers - -##### Primary node - -On primary database node edit `/etc/gitlab/gitlab.rb`: - -```ruby -# Disable all components except PostgreSQL, Repmgr, and Consul -roles ['postgres_role'] - -# PostgreSQL configuration -postgresql['listen_address'] = '0.0.0.0' -postgresql['hot_standby'] = 'on' -postgresql['wal_level'] = 'replica' -postgresql['shared_preload_libraries'] = 'repmgr_funcs' - -# Disable automatic database migrations -gitlab_rails['auto_migrate'] = false - -# Configure the Consul agent -consul['services'] = %w(postgresql) - -postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' -postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' -postgresql['max_wal_senders'] = 4 - -postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) -repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) - -consul['configuration'] = { - server: true, - retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23) -} -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -###### Secondary nodes - -On secondary nodes, edit `/etc/gitlab/gitlab.rb` and add all the information added -to primary node, noted above. In addition, append the following configuration - -```ruby -# HA setting to specify if a node should attempt to be master on initialization -repmgr['master_on_initialization'] = false -``` - -##### Example minimal configuration for application server - -On the server edit `/etc/gitlab/gitlab.rb`: - -```ruby -external_url 'http://gitlab.example.com' - -gitlab_rails['db_host'] = '127.0.0.1' -gitlab_rails['db_port'] = 6432 -gitlab_rails['db_password'] = 'toomanysecrets' -gitlab_rails['auto_migrate'] = false - -postgresql['enable'] = false -pgbouncer['enable'] = true -consul['enable'] = true - -# Configure PgBouncer -pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - -# Configure Consul agent -consul['watchers'] = %w(postgresql) - -pgbouncer['users'] = { - 'gitlab-consul': { - password: '5e0e3263571e3704ad655076301d6ebe' - }, - 'pgbouncer': { - password: '771a8625958a529132abe6f1a4acb19c' - } -} - -consul['configuration'] = { - retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23) -} -``` - -[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -##### Example minimal setup manual steps - -The manual steps for this configuration are the same as for the [example recommended setup](#example-recommended-setup-manual-steps). - -#### Failover procedure - -By default, if the master database fails, `repmgrd` should promote one of the -standby nodes to master automatically, and Consul will update PgBouncer with -the new master. - -If you need to failover manually, you have two options: - -**Shutdown the current master database** - -Run: - -```shell -gitlab-ctl stop postgresql -``` - -The automated failover process will see this and failover to one of the -standby nodes. - -**Or perform a manual failover** - -1. Ensure the old master node is not still active. -1. Login to the server that should become the new master and run: - - ```shell - gitlab-ctl repmgr standby promote - ``` - -1. If there are any other standby servers in the cluster, have them follow - the new master server: - - ```shell - gitlab-ctl repmgr standby follow NEW_MASTER - ``` - -#### Restore procedure - -If a node fails, it can be removed from the cluster, or added back as a standby -after it has been restored to service. - -##### Remove a standby from the cluster - - From any other node in the cluster, run: - - ```shell - gitlab-ctl repmgr standby unregister --node=X - ``` - - where X is the value of node in `repmgr.conf` on the old server. - - To find this, you can use: - - ```shell - awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf - ``` - - It will output something like: - - ```plaintext - 959789412 - ``` - - Then you will use this ID to unregister the node: - - ```shell - gitlab-ctl repmgr standby unregister --node=959789412 - ``` - -##### Add a node as a standby server - - From the stnadby node, run: - - ```shell - gitlab-ctl repmgr standby follow NEW_MASTER - gitlab-ctl restart repmgrd - ``` - - CAUTION: **Warning:** When the server is brought back online, and before - you switch it to a standby node, repmgr will report that there are two masters. - If there are any clients that are still attempting to write to the old master, - this will cause a split, and the old master will need to be resynced from - scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. - -##### Add a failed master back into the cluster as a standby node - - Once `repmgrd` and PostgreSQL are runnning, the node will need to follow the new - as a standby node. - - ```shell - gitlab-ctl repmgr standby follow NEW_MASTER - ``` - - Once the node is following the new master as a standby, the node needs to be - [unregistered from the cluster on the new master node](#remove-a-standby-from-the-cluster). - - Once the old master node has been unregistered from the cluster, it will need - to be setup as a new standby: - - ```shell - gitlab-ctl repmgr standby setup NEW_MASTER - ``` - - Failure to unregister and readd the old master node can lead to subsequent failovers - not working. - -#### Alternate configurations - -##### Database authorization - -By default, we give any host on the database network the permission to perform -repmgr operations using PostgreSQL's `trust` method. If you do not want this -level of trust, there are alternatives. - -You can trust only the specific nodes that will be database clusters, or you -can require md5 authentication. - -##### Trust specific addresses - -If you know the IP address, or FQDN of all database and PgBouncer nodes in the -cluster, you can trust only those nodes. - -In `/etc/gitlab/gitlab.rb` on all of the database nodes, set -`repmgr['trust_auth_cidr_addresses']` to an array of strings containing all of -the addresses. - -If setting to a node's FQDN, they must have a corresponding PTR record in DNS. -If setting to a node's IP address, specify it as `XXX.XXX.XXX.XXX/32`. - -For example: - -```ruby -repmgr['trust_auth_cidr_addresses'] = %w(192.168.1.44/32 db2.example.com) -``` - -##### MD5 Authentication - -If you are running on an untrusted network, repmgr can use md5 authentication -with a [`.pgpass` file](https://www.postgresql.org/docs/11/libpq-pgpass.html) -to authenticate. - -You can specify by IP address, FQDN, or by subnet, using the same format as in -the previous section: - -1. On the current master node, create a password for the `gitlab` and - `gitlab_repmgr` user: - - ```shell - gitlab-psql -d template1 - template1=# \password gitlab_repmgr - Enter password: **** - Confirm password: **** - template1=# \password gitlab - ``` - -1. On each database node: - - 1. Edit `/etc/gitlab/gitlab.rb`: - 1. Ensure `repmgr['trust_auth_cidr_addresses']` is **not** set - 1. Set `postgresql['md5_auth_cidr_addresses']` to the desired value - 1. Set `postgresql['sql_replication_user'] = 'gitlab_repmgr'` - 1. Reconfigure with `gitlab-ctl reconfigure` - 1. Restart PostgreSQL with `gitlab-ctl restart postgresql` - - 1. Create a `.pgpass` file. Enter the `gitlab_repmgr` password twice to - when asked: - - ```shell - gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' - ``` - -1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`: - 1. Ensure `gitlab_rails['db_password']` is set to the plaintext password for - the `gitlab` database user - 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect - -## Enable Monitoring - -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. - -If you enable Monitoring, it must be enabled on **all** database servers. - -1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: - - ```ruby - # Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true - - # Set the network addresses that the exporters will listen on - node_exporter['listen_address'] = '0.0.0.0:9100' - postgres_exporter['listen_address'] = '0.0.0.0:9187' - ``` - -1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. - -## Troubleshooting - -### Consul and PostgreSQL changes not taking effect - -Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it will not restart the services. However, not all changes can be activated by reloading. - -To restart either service, run `gitlab-ctl restart SERVICE` - -For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. - -On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](consul.md#restarting-the-server-cluster) for instructions on how to restart the service. - -### `gitlab-ctl repmgr-check-master` command produces errors - -If this command displays errors about database permissions it is likely that something failed during -install, resulting in the `gitlab-consul` database user getting incorrect permissions. Follow these -steps to fix the problem: - -1. On the master database node, connect to the database prompt - `gitlab-psql -d template1` -1. Delete the `gitlab-consul` user - `DROP USER "gitlab-consul";` -1. Exit the database prompt - `\q` -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) and the user will be re-added with the proper permissions. -1. Change to the `gitlab-consul` user - `su - gitlab-consul` -1. Try the check command again - `gitlab-ctl repmgr-check-master`. - -Now there should not be errors. If errors still occur then there is another problem. - -### PgBouncer error `ERROR: pgbouncer cannot connect to server` - -You may get this error when running `gitlab-rake gitlab:db:configure` or you -may see the error in the PgBouncer log file. - -```plaintext -PG::ConnectionBad: ERROR: pgbouncer cannot connect to server -``` - -The problem may be that your PgBouncer node's IP address is not included in the -`trust_auth_cidr_addresses` setting in `/etc/gitlab/gitlab.rb` on the database nodes. - -You can confirm that this is the issue by checking the PostgreSQL log on the master -database node. If you see the following error then `trust_auth_cidr_addresses` -is the problem. - -```plaintext -2018-03-29_13:59:12.11776 FATAL: no pg_hba.conf entry for host "123.123.123.123", user "pgbouncer", database "gitlabhq_production", SSL off -``` - -To fix the problem, add the IP address to `/etc/gitlab/gitlab.rb`. - -```ruby -postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>) -``` - -[Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -### Issues with other components - -If you're running into an issue with a component not outlined here, be sure to check the troubleshooting section of their specific documentation page. - -- [Consul](consul.md#troubleshooting) -- [PostgreSQL](https://docs.gitlab.com/omnibus/settings/database.html#troubleshooting) -- [GitLab application](gitlab.md#troubleshooting) - -## Configure using Omnibus - -**Note**: We recommend that you follow the instructions here for a full [PostgreSQL cluster](#high-availability-with-omnibus-gitlab-premium-only). -If you are reading this section due to an old bookmark, you can find that old documentation [in the repository](https://gitlab.com/gitlab-org/gitlab/blob/v10.1.4/doc/administration/high_availability/database.md#configure-using-omnibus). - -Read more on high-availability configuration: - -1. [Configure Redis](redis.md) -1. [Configure NFS](nfs.md) -1. [Configure the GitLab application servers](gitlab.md) -1. [Configure the load balancers](load_balancer.md) -1. [Manage the bundled Consul cluster](consul.md) +This content has been moved to a [new location](../postgresql/replication_and_failover.md). diff --git a/doc/administration/high_availability/gitaly.md b/doc/administration/high_availability/gitaly.md index 2e6bcabeb06..230f5baf33a 100644 --- a/doc/administration/high_availability/gitaly.md +++ b/doc/administration/high_availability/gitaly.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -11,7 +14,7 @@ This document is relevant for [scalable and highly available setups](../referenc ## Running Gitaly on its own server -See [Running Gitaly on its own server](../gitaly/index.md#running-gitaly-on-its-own-server) +See [Run Gitaly on its own server](../gitaly/index.md#run-gitaly-on-its-own-server) in Gitaly documentation. Continue configuration of other components by going back to the @@ -19,9 +22,9 @@ Continue configuration of other components by going back to the ## Enable Monitoring -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. -1. Make sure to collect [`CONSUL_SERVER_NODES`](database.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` +1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` 1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index cdafdbc4954..67a84f99bea 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -138,11 +138,11 @@ migrations performed. ## Enable Monitoring -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. If you enable Monitoring, it must be enabled on **all** GitLab servers. -1. Make sure to collect [`CONSUL_SERVER_NODES`](database.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` +1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` 1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: @@ -178,7 +178,7 @@ If you enable Monitoring, it must be enabled on **all** GitLab servers. running `sudo gitlab-ctl reconfigure`, it can take an extended period of time for Unicorn to complete reloading after receiving a `HUP`. For more information, see the - [issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4401). + [issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4401). ## Troubleshooting @@ -203,7 +203,7 @@ for more details. Read more on high-availability configuration: -1. [Configure the database](database.md) +1. [Configure the database](../postgresql/replication_and_failover.md) 1. [Configure Redis](redis.md) 1. [Configure NFS](nfs.md) 1. [Configure the load balancers](load_balancer.md) diff --git a/doc/administration/high_availability/load_balancer.md b/doc/administration/high_availability/load_balancer.md index beeb57a0e32..75703327140 100644 --- a/doc/administration/high_availability/load_balancer.md +++ b/doc/administration/high_availability/load_balancer.md @@ -2,9 +2,9 @@ type: reference --- -# Load Balancer for GitLab HA +# Load Balancer for multi-node GitLab -In an active/active GitLab configuration, you will need a load balancer to route +In an multi-node GitLab configuration, you will need a load balancer to route traffic to the application servers. The specifics on which load balancer to use or the exact configuration is beyond the scope of GitLab documentation. We hope that if you're managing HA systems like GitLab you have a load balancer of @@ -14,7 +14,7 @@ you need to use with GitLab. ## SSL -How will you handle SSL in your HA environment? There are several different +How will you handle SSL in your multi-node environment? There are several different options: - Each application node terminates SSL @@ -109,11 +109,15 @@ Configure DNS for an alternate SSH hostname such as `altssh.gitlab.example.com`. | ------- | ------------ | -------- | | 443 | 22 | TCP | +## Readiness check + +It is strongly recommend that multi-node deployments configure load balancers to utilize the [readiness check](../../user/admin_area/monitoring/health_check.md#readiness) to ensure a node is ready to accept traffic, before routing traffic to it. This is especially important when utilizing Puma, as there is a brief period during a restart where Puma will not accept requests. + --- Read more on high-availability configuration: -1. [Configure the database](database.md) +1. [Configure the database](../postgresql/replication_and_failover.md) 1. [Configure Redis](redis.md) 1. [Configure NFS](nfs.md) 1. [Configure the GitLab application servers](gitlab.md) diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index 409a4dfecb9..653a0b32ad7 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -4,7 +4,7 @@ type: reference # Configuring a Monitoring node for Scaling and High Availability -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. You can configure a Prometheus node to monitor GitLab. @@ -22,7 +22,7 @@ Omnibus: package you want using **steps 1 and 2** from the GitLab downloads page. - Do not complete any other steps on the download page. -1. Make sure to collect [`CONSUL_SERVER_NODES`](database.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` +1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index d2b8cf65b35..6511f9bd85d 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -59,7 +59,7 @@ management between systems: We recommend that all NFS users disable the NFS server delegation feature. This is to avoid a [Linux kernel bug](https://bugzilla.redhat.com/show_bug.cgi?id=1552203) which causes NFS clients to slow precipitously due to -[excessive network traffic from numerous `TEST_STATEID` NFS messages](https://gitlab.com/gitlab-org/gitlab-foss/issues/52017). +[excessive network traffic from numerous `TEST_STATEID` NFS messages](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52017). To disable NFS server delegation, do the following: @@ -146,7 +146,7 @@ Additionally, this configuration is specifically warned against in the >to the NFS server can cause data corruption problems. For supported database architecture, please see our documentation on -[Configuring a Database for GitLab HA](database.md). +[Configuring a Database for GitLab HA](../postgresql/replication_and_failover.md). ## NFS Client mount options @@ -164,7 +164,7 @@ Note there are several options that you should consider using: | Setting | Description | | ------- | ----------- | -| `vers=4.1` |NFS v4.1 should be used instead of v4.0 because there is a Linux [NFS client bug in v4.0](https://gitlab.com/gitlab-org/gitaly/issues/1339) that can cause significant problems due to stale data. +| `vers=4.1` |NFS v4.1 should be used instead of v4.0 because there is a Linux [NFS client bug in v4.0](https://gitlab.com/gitlab-org/gitaly/-/issues/1339) that can cause significant problems due to stale data. | `nofail` | Don't halt boot process waiting for this mount to become available | `lookupcache=positive` | Tells the NFS client to honor `positive` cache results but invalidates any `negative` cache results. Negative cache results cause problems with Git. Specifically, a `git push` can fail to register uniformly across all NFS clients. The negative cache causes the clients to 'remember' that the files did not exist previously. | `hard` | Instead of `soft`. [Further details](#soft-mount-option). @@ -194,7 +194,7 @@ use the `hard` option, because (from the man page): Other vendors make similar recommendations, including [SAP](http://wiki.scn.sap.com/wiki/x/PARnFQ) and NetApp's -[knowledge base](https://kb.netapp.com/app/answers/answer_view/a_id/1004893/~/hard-mount-vs-soft-mount-), +[knowledge base](https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/What_are_the_differences_between_hard_mount_and_soft_mount), they highlight that if the NFS client driver caches data, `soft` means there is no certainty if writes by GitLab are actually on disk. @@ -284,7 +284,7 @@ are empty before attempting a restore. Read more about the Read more on high-availability configuration: -1. [Configure the database](database.md) +1. [Configure the database](../postgresql/replication_and_failover.md) 1. [Configure Redis](redis.md) 1. [Configure the GitLab application servers](gitlab.md) 1. [Configure the load balancers](load_balancer.md) diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md index 4c672f49e26..15e4da5b1f7 100644 --- a/doc/administration/high_availability/pgbouncer.md +++ b/doc/administration/high_availability/pgbouncer.md @@ -12,124 +12,7 @@ In a HA setup, it's recommended to run a PgBouncer node separately for each data ### Running PgBouncer as part of an HA GitLab installation -1. Make sure you collect [`CONSUL_SERVER_NODES`](database.md#consul-information), [`CONSUL_PASSWORD_HASH`](database.md#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](database.md#pgbouncer-information) before executing the next step. - -1. One each node, edit the `/etc/gitlab/gitlab.rb` config file and replace values noted in the `# START user configuration` section as below: - - ```ruby - # Disable all components except PgBouncer and Consul agent - roles ['pgbouncer_role'] - - # Configure PgBouncer - pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - - # Configure Consul agent - consul['watchers'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # Replace CONSUL_PASSWORD_HASH with with a generated md5 value - # Replace PGBOUNCER_PASSWORD_HASH with with a generated md5 value - pgbouncer['users'] = { - 'gitlab-consul': { - password: 'CONSUL_PASSWORD_HASH' - }, - 'pgbouncer': { - password: 'PGBOUNCER_PASSWORD_HASH' - } - } - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - # - # END user configuration - ``` - - NOTE: **Note:** - `pgbouncer_role` was introduced with GitLab 10.3. - -1. Run `gitlab-ctl reconfigure` - -1. Create a `.pgpass` file so Consul is able to - reload PgBouncer. Enter the `PGBOUNCER_PASSWORD` twice when asked: - - ```shell - gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul - ``` - -#### PgBouncer Checkpoint - -1. Ensure each node is talking to the current master: - - ```shell - gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD - ``` - - If there is an error `psql: ERROR: Auth failed` after typing in the - password, ensure you previously generated the MD5 password hashes with the correct - format. The correct format is to concatenate the password and the username: - `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text - needed to generate an MD5 password hash for the `pgbouncer` user. - -1. Once the console prompt is available, run the following queries: - - ```shell - show databases ; show clients ; - ``` - - The output should be similar to the following: - - ```plaintext - name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections - ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- - gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 - pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 - (2 rows) - - type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls - ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- - C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | - (2 rows) - ``` - -#### Configure the internal load balancer - -If you're running more than one PgBouncer node as recommended, then at this time you'll need to set up a TCP internal load balancer to serve each correctly. This can be done with any reputable TCP load balancer. - -As an example here's how you could do it with [HAProxy](https://www.haproxy.org/): - -```plaintext -global - log /dev/log local0 - log localhost local1 notice - log stdout format raw local0 - -defaults - log global - default-server inter 10s fall 3 rise 2 - balance leastconn - -frontend internal-pgbouncer-tcp-in - bind *:6432 - mode tcp - option tcplog - - default_backend pgbouncer - -backend pgbouncer - mode tcp - option tcp-check - - server pgbouncer1 <ip>:6432 check - server pgbouncer2 <ip>:6432 check - server pgbouncer3 <ip>:6432 check -``` - -Refer to your preferred Load Balancer's documentation for further guidance. +This content has been moved to a [new location](../postgresql/replication_and_failover.md#configuring-the-pgbouncer-node). ### Running PgBouncer as part of a non-HA GitLab installation @@ -179,7 +62,7 @@ Refer to your preferred Load Balancer's documentation for further guidance. ## Enable Monitoring -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. If you enable Monitoring, it must be enabled on **all** PgBouncer servers. diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index d52c80aec0d..bad50f7ca74 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -746,11 +746,11 @@ gitlab_rails['redis_sentinels'] = [ ## Enable Monitoring -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. If you enable Monitoring, it must be enabled on **all** Redis servers. -1. Make sure to collect [`CONSUL_SERVER_NODES`](database.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` +1. Make sure to collect [`CONSUL_SERVER_NODES`](../postgresql/replication_and_failover.md#consul-information), which are the IP addresses or DNS records of the Consul server nodes, for the next step. Note they are presented as `Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z` 1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: @@ -1002,7 +1002,7 @@ Changes to Redis HA over time. Read more on High Availability: 1. [High Availability Overview](README.md) -1. [Configure the database](database.md) +1. [Configure the database](../postgresql/replication_and_failover.md) 1. [Configure NFS](nfs.md) 1. [Configure the GitLab application servers](gitlab.md) 1. [Configure the load balancers](load_balancer.md) diff --git a/doc/administration/housekeeping.md b/doc/administration/housekeeping.md index 4a2e2b9aac9..4110f8b7646 100644 --- a/doc/administration/housekeeping.md +++ b/doc/administration/housekeeping.md @@ -1,6 +1,6 @@ # Housekeeping -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/3041) in GitLab 8.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3041) in GitLab 8.4. ## Automatic housekeeping diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index fcd69464b13..e078a7c1098 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Incoming email GitLab has several features based on receiving incoming emails: @@ -92,13 +98,13 @@ authenticate solely based on access to an email domain such as `*.hooli.com.` Alternatively, use a dedicated domain for GitLab email communications such as `hooli-gitlab.com`. -See GitLab issue [#30366](https://gitlab.com/gitlab-org/gitlab-foss/issues/30366) +See GitLab issue [#30366](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30366) for a real-world example of this exploit. ### Omnibus package installations 1. Find the `incoming_email` section in `/etc/gitlab/gitlab.rb`, enable the feature - and fill in the details for your specific IMAP server and email account (see [examples](#config-examples) below). + and fill in the details for your specific IMAP server and email account (see [examples](#configuration-examples) below). 1. Reconfigure GitLab for the changes to take effect: @@ -124,7 +130,7 @@ Reply by email should now be working. ``` 1. Find the `incoming_email` section in `config/gitlab.yml`, enable the feature - and fill in the details for your specific IMAP server and email account (see [examples](#config-examples) below). + and fill in the details for your specific IMAP server and email account (see [examples](#configuration-examples) below). 1. Enable `mail_room` in the init script at `/etc/default/gitlab`: @@ -147,7 +153,7 @@ Reply by email should now be working. Reply by email should now be working. -### Config examples +### Configuration examples #### Postfix diff --git a/doc/administration/index.md b/doc/administration/index.md index fb827ae8573..fa415e5f78d 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -109,7 +109,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Sign-up restrictions](../user/admin_area/settings/sign_up_restrictions.md): block email addresses of specific domains, or whitelist only specific domains. - [Access restrictions](../user/admin_area/settings/visibility_and_access_controls.md#enabled-git-access-protocols): Define which Git access protocols can be used to talk to GitLab (SSH, HTTP, HTTPS). - [Authentication and Authorization](auth/README.md): Configure external authentication with LDAP, SAML, CAS, and additional providers. - - [Sync LDAP](auth/ldap-ee.md) **(STARTER ONLY)** + - [Sync LDAP](auth/ldap/index.md) **(STARTER ONLY)** - [Kerberos authentication](../integration/kerberos.md) **(STARTER ONLY)** - See also other [authentication](../topics/authentication/index.md#gitlab-administrators) topics (for example, enforcing 2FA). - [Email users](../tools/email.md): Email GitLab users from within GitLab. **(STARTER ONLY)** @@ -157,7 +157,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [External Pipeline Validation](external_pipeline_validation.md): Enable, disable and configure external pipeline validation. - [Job artifacts](job_artifacts.md): Enable, disable, and configure job artifacts (a set of files and directories which are outputted by a job when it completes successfully). - [Job logs](job_logs.md): Information about the job logs. -- [Register Shared and specific Runners](../ci/runners/README.md#registering-a-shared-runner): Learn how to register and configure Shared and specific Runners to your own instance. +- [Register Runners](../ci/runners/README.md#types-of-runners): Learn how to register and configure Runners. - [Shared Runners pipelines quota](../user/admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota-starter-only): Limit the usage of pipeline minutes for Shared Runners. **(STARTER ONLY)** - [Enable/disable Auto DevOps](../topics/autodevops/index.md#enablingdisabling-auto-devops): Enable or disable Auto DevOps for your instance. diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index d0e8f079cb2..3d2f7380494 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -10,7 +10,7 @@ performance, data, or could even exhaust the allocated resources for the applica ## Number of comments per issue, merge request or commit -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/22388) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22388) in GitLab 12.4. There's a limit to the number of comments that can be submitted on an issue, merge request, or commit. When the limit is reached, system notes can still be @@ -21,7 +21,7 @@ will fail. ## Size of comments and descriptions of issues, merge requests, and epics -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/61974) in GitLab 12.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61974) in GitLab 12.2. There is a limit to the size of comments and descriptions of issues, merge requests, and epics. Attempting to add a body of text larger than the limit will result in an error, and the @@ -43,7 +43,7 @@ When the number exceeds the limit the page displays an alert and links to a pagi ## Number of pipelines per Git push -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/51401) in GitLab 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51401) in GitLab 11.10. The number of pipelines that can be created in a single push is 4. This is to prevent the accidental creation of pipelines when `git push --all` @@ -53,9 +53,9 @@ Read more in the [CI documentation](../ci/yaml/README.md#processing-git-pushes). ## Retention of activity history -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/21164) in GitLab 8.12. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/21164) in GitLab 8.12. -Activity history for projects and individuals' profiles was limited to one year until [GitLab 11.4](https://gitlab.com/gitlab-org/gitlab-foss/issues/52246) when it was extended to two years, and in [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/33840) to three years. +Activity history for projects and individuals' profiles was limited to one year until [GitLab 11.4](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52246) when it was extended to two years, and in [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/33840) to three years. ## Number of embedded metrics @@ -87,7 +87,7 @@ NOTE: **Note:** Set the limit to `0` to disable it. ## Incoming emails from auto-responders -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30327) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30327) in GitLab 12.4. GitLab ignores all incoming emails sent from auto-responders by looking for the `X-Autoreply` header. Such emails don't create comments on issues or merge requests. @@ -126,7 +126,7 @@ NOTE: **Note:** Set the limit to `0` to disable it. ### Number of jobs in active pipelines -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32823) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32823) in GitLab 12.6. The total number of jobs in active pipelines can be limited per project. This limit is checked each time a new pipeline is created. An active pipeline is any pipeline in one of the following states: @@ -156,7 +156,7 @@ NOTE: **Note:** Set the limit to `0` to disable it. ### Number of CI/CD subscriptions to a project -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9045) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9045) in GitLab 12.9. The total number of subscriptions can be limited per project. This limit is checked each time a new subscription is created. @@ -199,6 +199,24 @@ To set this limit on a self-managed installation, run the following in the Plan.default.limits.update!(ci_pipeline_schedules: 100) ``` +### Number of instance level variables + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216097) in GitLab 13.1. + +The total number of instance level CI/CD variables is limited at the instance level. +This limit is checked each time a new instance level variable is created. If a new variable +would cause the total number of variables to exceed the limit, the new variable will not be created. + +On self-managed instances this limit is defined for the `default` plan. By default, +this limit is set to `25`. + +To update this limit to a new value on a self-managed installation, run the following in the +[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session): + +```ruby +Plan.default.limits.update!(ci_instance_level_variables: 30) +``` + ## Instance monitoring and metrics ### Incident Management inbound alert limits @@ -235,7 +253,7 @@ Kubernetes won't be shown. Reports that go over the 20 MB limit won't be loaded. Affected reports: -- [Merge Request security reports](../user/project/merge_requests/index.md#security-reports-ultimate) +- [Merge Request security reports](../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports-ultimate) - [CI/CD parameter `artifacts:expose_as`](../ci/yaml/README.md#artifactsexpose_as) - [JUnit test reports](../ci/junit_test_reports.md) @@ -243,7 +261,7 @@ Reports that go over the 20 MB limit won't be loaded. Affected reports: ### Maximum field length -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201826) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201826) in GitLab 12.8. You can set a limit on the content of text fields indexed for Global Search. Setting a maximum helps to reduce the load of the indexing processes. If any @@ -270,7 +288,7 @@ See the [documentation on Snippets settings](snippets/index.md). ### Webhooks and Project Services -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31009) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31009) in GitLab 12.4. Total number of changes (branches or tags) in a single push. If changes are more than the specified limit, hooks won't be executed. @@ -282,7 +300,7 @@ More information can be found in these docs: ### Activities -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31007) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31007) in GitLab 12.4. Total number of changes (branches or tags) in a single push to determine whether individual push events or bulk push event will be created. diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 3372d05bd6b..cd25fbf351b 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -105,6 +105,21 @@ To activate the changes, run the following command: sudo gitlab-ctl reconfigure ``` +### Security + +PlantUML has features that allows fetching network resources. + +```plaintext +@startuml +start + ' ... + !include http://localhost/ +stop; +@enduml +``` + +**If you self-host the PlantUML server, network controls should be put in place to isolate it.** + ## GitLab You need to enable PlantUML integration from Settings under Admin Area. To do diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index fbfad46ef65..0dbda67d39b 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -1,3 +1,10 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference, howto +--- + # Jobs artifacts administration > - Introduced in GitLab 8.2 and GitLab Runner 0.7.0. @@ -40,6 +47,9 @@ GitLab Runner can upload an archive containing the job artifacts to GitLab. By d this is done when the job succeeds, but can also be done on failure, or always, via the [`artifacts:when`](../ci/yaml/README.md#artifactswhen) parameter. +Most artifacts are compressed by GitLab Runner before being sent to the coordinator. The exception to this is +[reports artifacts](../ci/pipelines/job_artifacts.md#artifactsreports), which are compressed after uploading. + ### Using local storage To change the location where the artifacts are stored locally, follow the steps @@ -119,7 +129,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | | `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | | `region` | AWS region | us-east-1 | -| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | +| `host` | S3 compatible host for when not using AWS, for example `localhost` or `storage.example.com` | s3.amazonaws.com | | `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | | `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | | `use_iam_profile` | Set to true to use IAM profile instead of access keys | false @@ -129,7 +139,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a _The artifacts are stored by default in `/var/opt/gitlab/gitlab-rails/shared/artifacts`._ -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with +1. Edit `/etc/gitlab/gitlab.rb` and add the following lines, substituting the values you want: ```ruby @@ -144,7 +154,7 @@ _The artifacts are stored by default in } ``` - NOTE: For GitLab 9.4+, if you are using AWS IAM profiles, be sure to omit the + NOTE: For GitLab 9.4+, if you're using AWS IAM profiles, be sure to omit the AWS access key and secret access key/value pairs. For example: ```ruby @@ -164,7 +174,7 @@ _The artifacts are stored by default in CAUTION: **CAUTION:** JUnit test report artifact (`junit.xml.gz`) migration -[is not supported](https://gitlab.com/gitlab-org/gitlab/issues/27698) +[is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/27698) by the `gitlab:artifacts:migrate` script. **In installations from source:** @@ -197,7 +207,7 @@ _The artifacts are stored by default in CAUTION: **CAUTION:** JUnit test report artifact (`junit.xml.gz`) migration -[is not supported](https://gitlab.com/gitlab-org/gitlab/issues/27698) +[is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/27698) by the `gitlab:artifacts:migrate` script. ### OpenStack compatible connection settings @@ -209,8 +219,8 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | `provider` | Always `OpenStack` for compatible hosts | OpenStack | | `openstack_username` | OpenStack username | | | `openstack_api_key` | OpenStack API key | | -| `openstack_temp_url_key` | OpenStack key for generating temporary urls | | -| `openstack_auth_url` | OpenStack authentication endpont | | +| `openstack_temp_url_key` | OpenStack key for generating temporary URLs | | +| `openstack_auth_url` | OpenStack authentication endpoint | | | `openstack_region` | OpenStack region | | | `openstack_tenant_id` | OpenStack tenant ID | @@ -219,7 +229,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a _The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/shared/artifacts`._ -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with +1. Edit `/etc/gitlab/gitlab.rb` and add the following lines, substituting the values you want: ```ruby @@ -296,17 +306,20 @@ In order to migrate back to local storage: ## Expiring artifacts -If an expiry date is used for the artifacts, they are marked for deletion -right after that date passes. Artifacts are cleaned up by the -`expire_build_artifacts_worker` cron job which is run by Sidekiq every hour at -50 minutes (`50 * * * *`). +If [`artifacts:expire_in`](../ci/yaml/README.md#artifactsexpire_in) is used to set +an expiry for the artifacts, they are marked for deletion right after that date passes. +Otherwise, they will expire per the [default artifacts expiration setting](../user/admin_area/settings/continuous_integration.md). + +Artifacts are cleaned up by the `expire_build_artifacts_worker` cron job which Sidekiq +runs every hour at 50 minutes (`50 * * * *`). To change the default schedule on which the artifacts are expired, follow the steps below. **In Omnibus installations:** -1. Edit `/etc/gitlab/gitlab.rb` and comment out or add the following line +1. Edit `/etc/gitlab/gitlab.rb` and add the following line (or uncomment it if it already exists and is commented out), substituting + your schedule in cron syntax: ```ruby gitlab_rails['expire_build_artifacts_worker_cron'] = "50 * * * *" @@ -326,12 +339,15 @@ steps below. 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. +If the `expire` directive is not set explicitly in your pipeline, artifacts will expire per the +default artifacts expiration setting, which you can find in the [CI/CD Admin settings](../user/admin_area/settings/continuous_integration.md). + ## Validation for dependencies > Introduced in GitLab 10.3. To disable [the dependencies validation](../ci/yaml/README.md#when-a-dependent-job-will-fail), -you can flip the feature flag from a Rails console. +you can enable the `ci_disable_validates_dependencies` feature flag from a Rails console. **In Omnibus installations:** @@ -341,10 +357,10 @@ you can flip the feature flag from a Rails console. sudo gitlab-rails console ``` -1. Flip the switch and disable it: +1. Enable the feature flag to disable the validation: ```ruby - Feature.enable('ci_disable_validates_dependencies') + Feature.enable(:ci_disable_validates_dependencies) ``` **In installations from source:** @@ -356,15 +372,15 @@ you can flip the feature flag from a Rails console. sudo -u git -H bundle exec rails console -e production ``` -1. Flip the switch and disable it: +1. Enable the feature flag to disable the validation: ```ruby - Feature.enable('ci_disable_validates_dependencies') + Feature.enable(:ci_disable_validates_dependencies) ``` ## Set the maximum file size of the artifacts -Provided the artifacts are enabled, you can change the maximum file size of the +If artifacts are enabled, you can change the maximum file size of the artifacts through the [Admin Area settings](../user/admin_area/settings/continuous_integration.md#maximum-artifacts-size-core-only). ## Storage statistics @@ -404,6 +420,8 @@ In these and other cases, you'll need to identify the projects most responsible for disk space usage, figure out what types of artifacts are using the most space, and in some cases, manually delete job artifacts to reclaim disk space. +One possible first step is to [clean up _orphaned_ artifact files](../raketasks/cleanup.md#remove-orphan-artifact-files). + #### List projects by total size of job artifacts stored List the top 20 projects, sorted by the total size of job artifacts stored, by @@ -488,7 +506,7 @@ highly recommend running them only under the guidance of a Support Engineer, or running them in a test environment with a backup of the instance ready to be restored, just in case. -If you need to manually remove ALL job artifacts associated with multiple jobs, +If you need to manually remove **all** job artifacts associated with multiple jobs, **including job logs**, this can be done from the Rails console (`sudo gitlab-rails console`): 1. Select jobs to be deleted: diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index 5c03ff1f4b2..d3484536a76 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -1,6 +1,13 @@ +--- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + # Job logs -> [Renamed from job traces to job logs](https://gitlab.com/gitlab-org/gitlab/issues/29121) in GitLab 12.5. +> [Renamed from job traces to job logs](https://gitlab.com/gitlab-org/gitlab/-/issues/29121) in GitLab 12.5. Job logs are sent by GitLab Runner while it's processing a job. You can see logs in job pages, pipelines, email notifications, etc. @@ -64,10 +71,19 @@ There isn't a way to automatically expire old job logs, but it's safe to remove them if they're taking up too much space. If you remove the logs manually, the job output in the UI will be empty. +For example, to delete all job logs older than 60 days, run the following from a shell in your GitLab instance: + +DANGER: **Warning:** +This command will permanently delete the log files and is irreversible. + +```shell +find /var/opt/gitlab/gitlab-rails/shared/artifacts -name "job.log" -mtime +60 -delete +``` + ## New incremental logging architecture > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18169) in GitLab 10.4. -> - [Announced as generally available](https://gitlab.com/gitlab-org/gitlab-foss/issues/46097) in GitLab 11.0. +> - [Announced as generally available](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/46097) in GitLab 11.0. NOTE: **Note:** This feature is off by default. See below for how to [enable or disable](#enabling-incremental-logging) it. @@ -112,13 +128,13 @@ sudo -u git -H bin/rails console -e production **To check if incremental logging (trace) is enabled:** ```ruby -Feature.enabled?('ci_enable_live_trace') +Feature.enabled?(:ci_enable_live_trace) ``` **To enable incremental logging (trace):** ```ruby -Feature.enable('ci_enable_live_trace') +Feature.enable(:ci_enable_live_trace) ``` NOTE: **Note:** diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index e2b982448ef..dd0e25b05f1 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -110,7 +110,7 @@ Here is a configuration example with GCS. | `provider` | The provider name | `Google` | | `google_project` | GCP project name | `gcp-project-12345` | | `google_client_email` | The email address of the service account | `foo@gcp-project-12345.iam.gserviceaccount.com` | -| `google_json_key_location` | The json key path | `/path/to/gcp-project-12345-abcde.json` | +| `google_json_key_location` | The JSON key path | `/path/to/gcp-project-12345-abcde.json` | NOTE: **Note:** The service account must have permission to access the bucket. diff --git a/doc/administration/libravatar.md b/doc/administration/libravatar.md index e1c38b3409f..beecd9e4783 100644 --- a/doc/administration/libravatar.md +++ b/doc/administration/libravatar.md @@ -37,7 +37,7 @@ the configuration options as follows: ### Your own Libravatar server -If you are [running your own libravatar service](https://wiki.libravatar.org/running_your_own/), +If you are [running your own Libravatar service](https://wiki.libravatar.org/running_your_own/), the URL will be different in the configuration, but you must provide the same placeholders so GitLab can parse the URL correctly. diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 57b12897979..7d7053a26db 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -46,6 +46,8 @@ Line breaks have been added to this example for legibility: "gitaly_duration_s":0.16, "redis_calls":115, "redis_duration_s":0.13, + "redis_read_bytes":1507378, + "redis_write_bytes":2920, "correlation_id":"O1SdybnnIq7", "cpu_s":17.50, "db_duration_s":0.08, @@ -56,23 +58,60 @@ Line breaks have been added to this example for legibility: This example was a GET request for a specific issue. Each line also contains performance data, with times in -milliseconds: +seconds: 1. `duration_s`: total time taken to retrieve the request 1. `queue_duration_s`: total time that the request was queued inside GitLab Workhorse 1. `view_duration_s`: total time taken inside the Rails views 1. `db_duration_s`: total time to retrieve data from PostgreSQL -1. `redis_duration_s`: total time to retrieve data from Redis 1. `cpu_s`: total time spent on CPU 1. `gitaly_duration_s`: total time taken by Gitaly calls 1. `gitaly_calls`: total number of calls made to Gitaly 1. `redis_calls`: total number of calls made to Redis +1. `redis_duration_s`: total time to retrieve data from Redis +1. `redis_read_bytes`: total bytes read from Redis +1. `redis_write_bytes`: total bytes written to Redis +1. `redis_<instance>_calls`: total number of calls made to a Redis instance +1. `redis_<instance>_duration_s`: total time to retrieve data from a Redis instance +1. `redis_<instance>_read_bytes`: total bytes read from a Redis instance +1. `redis_<instance>_write_bytes`: total bytes written to a Redis instance User clone and fetch activity using HTTP transport appears in this log as `action: git_upload_pack`. In addition, the log contains the originating IP address, (`remote_ip`),the user's ID (`user_id`), and username (`username`). +Some endpoints such as `/search` may make requests to Elasticsearch if using +[Advanced Global Search](../user/search/advanced_global_search.md). These will +additionally log `elasticsearch_calls` and `elasticsearch_call_duration_s`, +which correspond to: + +1. `elasticsearch_calls`: total number of calls to Elasticsearch +1. `elasticsearch_duration_s`: total time taken by Elasticsearch calls + +ActionCable connection and subscription events are also logged to this file and they follow the same +format above. The `method`, `path`, and `format` fields are not applicable, and are always empty. +The ActionCable connection or channel class is used as the `controller`. + +```json +{ + "method":{}, + "path":{}, + "format":{}, + "controller":"IssuesChannel", + "action":"subscribe", + "status":200, + "time":"2020-05-14T19:46:22.008Z", + "params":[{"key":"project_path","value":"gitlab/gitlab-foss"},{"key":"iid","value":"1"}], + "remote_ip":"127.0.0.1", + "user_id":1, + "username":"admin", + "ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:76.0) Gecko/20100101 Firefox/76.0", + "correlation_id":"jSOIEynHCUa", + "duration_s":0.32566 +} +``` + NOTE: **Note:** Starting with GitLab 12.5, if an error occurs, an `exception` field is included with `class`, `message`, and `backtrace`. Previous versions included an `error` field instead of @@ -218,7 +257,7 @@ October 07, 2014 11:25: Project "project133" was removed ## `application_json.log` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/22812) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22812) in GitLab 12.7. This file lives in `/var/log/gitlab/gitlab-rails/application_json.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/application_json.log` for @@ -340,7 +379,7 @@ only. For example: ## `audit_json.log` NOTE: **Note:** -Most log entries only exist in [GitLab Starter](https://about.gitlab.com/pricing), however a few exist in GitLab Core. +Most log entries only exist in [GitLab Starter](https://about.gitlab.com/pricing/), however a few exist in GitLab Core. This file lives in `/var/log/gitlab/gitlab-rails/audit_json.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/audit_json.log` for @@ -593,6 +632,16 @@ installations from source. It logs the progress of the import process. +## `exporter.log` + +> Introduced in GitLab 13.1. + +This file lives in `/var/log/gitlab/gitlab-rails/exporter.log` for +Omnibus GitLab packages or in `/home/git/gitlab/log/exporter.log` for +installations from source. + +It logs the progress of the export process. + ## `auth.log` > Introduced in GitLab 12.0. @@ -608,12 +657,12 @@ This log records: - [Protected paths](../user/admin_area/settings/protected_paths.md) abusive requests. NOTE: **Note:** -From [%12.1](https://gitlab.com/gitlab-org/gitlab-foss/issues/62756), user ID and username are also +From [%12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239), user ID and username are also recorded on this log, if available. ## `graphql_json.log` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/59587) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59587) in GitLab 12.0. This file lives in `/var/log/gitlab/gitlab-rails/graphql_json.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/graphql_json.log` for @@ -724,17 +773,6 @@ Each line contains a JSON line that can be ingested by Elasticsearch. For exampl "severity": "ERROR", "time": "2019-12-17T11:49:29.485Z", "correlation_id": "AbDVUrrTvM1", - "extra.server": { - "os": { - "name": "Darwin", - "version": "Darwin Kernel Version 19.2.0", - "build": "19.2.0", - }, - "runtime": { - "name": "ruby", - "version": "ruby 2.6.5p114 (2019-10-01 revision 67812) [x86_64-darwin18]" - } - }, "extra.project_id": 55, "extra.relation_key": "milestones", "extra.relation_index": 1, @@ -756,7 +794,7 @@ Omnibus GitLab packages or in `/home/git/gitlab/log/service_measurement.log` for installations from source. It contain only a single structured log with measurements for each service execution. -It will contain measurement such as: number of sql calls, execution_time, gc_stats, memory usage, etc... +It will contain measurement such as: number of SQL calls, `execution_time`, `gc_stats`, memory usage, etc... For example: diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 795933e2772..da9ca42ed9e 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -1,6 +1,6 @@ # Merge request diffs storage **(CORE ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/52568) in GitLab 11.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52568) in GitLab 11.8. Merge request diffs are size-limited copies of diffs associated with merge requests. When viewing a merge request, diffs are sourced from these copies diff --git a/doc/administration/monitoring/github_imports.md b/doc/administration/monitoring/github_imports.md index 6d5915930b2..21cc4c708a8 100644 --- a/doc/administration/monitoring/github_imports.md +++ b/doc/administration/monitoring/github_imports.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring GitHub imports >**Note:** diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index 091da46f316..d05fb803c5c 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -1,7 +1,13 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab self monitoring project -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32351) in GitLab 12.7, behind a disabled feature flag (`self_monitoring_project`). -> - The feature flag was removed and the Self Monitoring Project was [made generally available](https://gitlab.com/gitlab-org/gitlab/issues/198511) in GitLab 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32351) in GitLab 12.7, behind a disabled feature flag (`self_monitoring_project`). +> - The feature flag was removed and the Self Monitoring Project was [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/198511) in GitLab 12.8. GitLab has been adding the ability for administrators to see insights into the health of their GitLab instance. In order to surface this experience in a native way, similar to how @@ -15,10 +21,12 @@ All administrators at the time of creation of the project and group will be adde as maintainers of the group and project, and as an admin, you'll be able to add new members to the group in order to give them maintainer access to the project. -This project is used to self monitor your GitLab instance. Metrics are not yet -fully integrated, and the dashboard does not aggregate any data on Omnibus installations. GitLab plans -to provide integrated self-monitoring metrics in a future release. You can -currently use the project to configure your own [custom metrics](../../../user/project/integrations/prometheus.md#adding-custom-metrics) using +This project is used to self monitor your GitLab instance. The metrics dashboard +of the project shows some basic resource usage charts, such as CPU and memory usage +of each server in [Omnibus GitLab](https://docs.gitlab.com/omnibus/) installations. + +You can also use the project to configure your own +[custom metrics](../../../user/project/integrations/prometheus.md#adding-custom-metrics) using metrics exposed by the [GitLab exporter](../prometheus/gitlab_metrics.md#metrics-available). ## Creating the self monitoring project @@ -53,7 +61,7 @@ you should ## Taking action on Prometheus alerts **(ULTIMATE)** You can [add a webhook](../../../user/project/integrations/prometheus.md#external-prometheus-instances) -to the Prometheus config in order for GitLab to receive notifications of any alerts. +to the Prometheus configuration in order for GitLab to receive notifications of any alerts. Once the webhook is setup, you can [take action on incoming alerts](../../../user/project/integrations/prometheus.md#taking-action-on-incidents-ultimate). @@ -69,7 +77,7 @@ You can add custom metrics in the self monitoring project by: ### Getting error message in logs: `Could not create instance administrators group. Errors: ["You don’t have permission to create groups."]` -There is [a bug](https://gitlab.com/gitlab-org/gitlab/issues/208676) which causes +There is [a bug](https://gitlab.com/gitlab-org/gitlab/-/issues/208676) which causes project creation to fail with the following error (which appears in the log file) when the first admin user is an [external user](../../../user/permissions.md#external-users-core-only): diff --git a/doc/administration/monitoring/index.md b/doc/administration/monitoring/index.md index 1d7c52a198f..a54c25450c6 100644 --- a/doc/administration/monitoring/index.md +++ b/doc/administration/monitoring/index.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Monitoring GitLab Explore our features to monitor your GitLab instance: diff --git a/doc/administration/monitoring/ip_whitelist.md b/doc/administration/monitoring/ip_whitelist.md index 1b0ea1744d9..862a9368be8 100644 --- a/doc/administration/monitoring/ip_whitelist.md +++ b/doc/administration/monitoring/ip_whitelist.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # IP whitelist > Introduced in GitLab 9.4. diff --git a/doc/administration/monitoring/performance/gitlab_configuration.md b/doc/administration/monitoring/performance/gitlab_configuration.md index 14119a5d8f3..d09dabab40d 100644 --- a/doc/administration/monitoring/performance/gitlab_configuration.md +++ b/doc/administration/monitoring/performance/gitlab_configuration.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Configuration GitLab Performance Monitoring is disabled by default. To enable it and change any of its diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 3438a564d2f..4dbe3aed84e 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Grafana Configuration [Grafana](https://grafana.com/) is a tool that allows you to visualize time @@ -57,7 +63,7 @@ repository for more information on this process. ## Integration with GitLab UI -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/61005) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61005) in GitLab 12.1. If you have set up Grafana, you can enable a link to access it easily from the sidebar: diff --git a/doc/administration/monitoring/performance/img/request_profiling_token.png b/doc/administration/monitoring/performance/img/request_profiling_token.png Binary files differdeleted file mode 100644 index ee819fcb437..00000000000 --- a/doc/administration/monitoring/performance/img/request_profiling_token.png +++ /dev/null diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md index 02070287611..6f22327e499 100644 --- a/doc/administration/monitoring/performance/index.md +++ b/doc/administration/monitoring/performance/index.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Performance Monitoring GitLab comes with its own application performance measuring system as of GitLab diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index 89246bc9782..c681dedbca8 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Performance Bar A Performance Bar can be displayed, to dig into the performance of a page. When @@ -18,7 +24,7 @@ It allows you to see (from left to right):  - total load timings of the page; click through for details of these calls. Values in the following order: - Backend - Time that the actual base page took to load - - [First Contentful Paint](https://developers.google.com/web/tools/lighthouse/audits/first-contentful-paint) - Time until something was visible to the user + - [First Contentful Paint](hhttps://web.dev/first-contentful-paint/) - Time until something was visible to the user - [DomContentLoaded](https://developers.google.com/web/fundamentals/performance/critical-rendering-path/measure-crp) Event - Number of Requests that the page loaded  diff --git a/doc/administration/monitoring/performance/request_profiling.md b/doc/administration/monitoring/performance/request_profiling.md index c32edb60f9d..a3b29493d84 100644 --- a/doc/administration/monitoring/performance/request_profiling.md +++ b/doc/administration/monitoring/performance/request_profiling.md @@ -1,17 +1,39 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Request Profiling -## Procedure +To profile a request: + +1. Sign in to GitLab as a user with Administrator or Maintainer [permissions](../../../user/permissions.md). +1. In the navigation bar, click **{admin}** **Admin area**. +1. Navigate to **{monitor}** **Monitoring > Requests Profiles**. +1. In the **Requests Profiles** section, copy the token. +1. Pass the headers `X-Profile-Token: <token>` and `X-Profile-Mode: <mode>`(where + `<mode>` can be `execution` or `memory`) to the request you want to profile. When + passing headers, you can use: + + - Browser extensions such as the + [ModHeader](https://chrome.google.com/webstore/detail/modheader/idgpnmonknjnojddfkpgkljpfnnfcklj) + Chrome extension. + - `curl`. For example: + + ```shell + curl --header 'X-Profile-Token: <token>' --header 'X-Profile-Mode: <mode>' "https://gitlab.example.com/group/project" + ``` + + NOTE: **Note:** + Profiled requests can take longer than usual. + +After the request completes, you can view the profiling output from the +**{monitor}** **Monitoring > Requests Profiles** administration page: -1. Grab the profiling token from **Monitoring > Requests Profiles** admin page - (highlighted in a blue in the image below). -  -1. Pass the header `X-Profile-Token: <token>` and `X-Profile-Mode: <mode>`(where `<mode>` can be `execution` or `memory`) to the request you want to profile. You can use: - - Browser extensions. For example, [ModHeader](https://chrome.google.com/webstore/detail/modheader/idgpnmonknjnojddfkpgkljpfnnfcklj) Chrome extension. - - `curl`. For example, `curl --header 'X-Profile-Token: <token>' --header 'X-Profile-Mode: <mode>' https://gitlab.example.com/group/project`. -1. Once request is finished (which will take a little longer than usual), you can - view the profiling output from **Monitoring > Requests Profiles** admin page. -  + -## Cleaning up +## Cleaning up profiled requests -Profiling output will be cleared out every day via a Sidekiq worker. +The output from profiled requests is cleared out once each day through a +Sidekiq worker. diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index f725db9a039..f3084b1732e 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -6,30 +6,29 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Prometheus metrics ->**Note:** -Available since [Omnibus GitLab 9.3](https://gitlab.com/gitlab-org/gitlab-foss/issues/29118). For -installations from source you'll have to configure it yourself. - To enable the GitLab Prometheus metrics: -1. Log into GitLab as an administrator. -1. Navigate to **Admin Area > Settings > Metrics and profiling**. +1. Log into GitLab as a user with [administrator permissions](../../../user/permissions.md). +1. Navigate to **{admin}** **Admin Area > Settings > Metrics and profiling**. 1. Find the **Metrics - Prometheus** section, and click **Enable Prometheus Metrics**. 1. [Restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart) for the changes to take effect. +NOTE: **Note:** +For installations from source you'll have to configure it yourself. + ## Collecting the metrics GitLab monitors its own internal service metrics, and makes them available at the -`/-/metrics` endpoint. Unlike other [Prometheus](https://prometheus.io) exporters, in order to access -it, the client IP needs to be [included in a whitelist](../ip_whitelist.md). +`/-/metrics` endpoint. Unlike other [Prometheus](https://prometheus.io) exporters, to access +it, the client IP address needs to be [explicitly allowed](../ip_whitelist.md). -For Omnibus and Chart installations, these metrics are automatically enabled -and collected as of [GitLab -9.4](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1702). For -source installations or earlier versions, these metrics will need to be enabled +For [Omnibus GitLab](https://docs.gitlab.com/omnibus/) and Chart installations, +these metrics are enabled and collected as of +[GitLab 9.4](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1702). +For source installations or earlier versions, these metrics must be enabled manually and collected by a Prometheus server. -See also [Sidekiq metrics](#sidekiq-metrics) for how to enable and view metrics from Sidekiq nodes. +For enabling and viewing metrics from Sidekiq nodes, see [Sidekiq metrics](#sidekiq-metrics). ## Metrics available @@ -43,32 +42,33 @@ The following metrics are available: | `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | | | `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller/action | `controller`, `action`, `operation` | | `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | | +| `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` | | `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` | | `job_waiter_timeouts_total` | Counter | 12.9 | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete | `worker` | | `gitlab_database_transaction_seconds` | Histogram | 12.1 | Time spent in database transactions, in seconds | | | `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | `controller`, `action`, `module`, `method` | | `gitlab_page_out_of_bounds` | Counter | 12.8 | Counter for the PageLimiter pagination limit being hit | `controller`, `action`, `bot` | | `gitlab_rails_queue_duration_seconds` | Histogram | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | | -| `gitlab_sql_duration_seconds` | Histogram | 10.2 | SQL execution time, excluding SCHEMA operations and BEGIN / COMMIT | | -| `gitlab_transaction_allocated_memory_bytes` | Histogram | 10.2 | Allocated memory for all transactions (gitlab_transaction_* metrics) | | +| `gitlab_sql_duration_seconds` | Histogram | 10.2 | SQL execution time, excluding `SCHEMA` operations and `BEGIN` / `COMMIT` | | +| `gitlab_transaction_allocated_memory_bytes` | Histogram | 10.2 | Allocated memory for all transactions (`gitlab_transaction_*` metrics) | | | `gitlab_transaction_cache_<key>_count_total` | Counter | 10.2 | Counter for total Rails cache calls (per key) | | | `gitlab_transaction_cache_<key>_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (per key) | | | `gitlab_transaction_cache_count_total` | Counter | 10.2 | Counter for total Rails cache calls (aggregate) | | | `gitlab_transaction_cache_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (aggregate) | | | `gitlab_transaction_cache_read_hit_count_total` | Counter | 10.2 | Counter for cache hits for Rails cache calls | `controller`, `action` | | `gitlab_transaction_cache_read_miss_count_total` | Counter | 10.2 | Counter for cache misses for Rails cache calls | `controller`, `action` | -| `gitlab_transaction_duration_seconds` | Histogram | 10.2 | Duration for all transactions (gitlab_transaction_* metrics) | `controller`, `action` | +| `gitlab_transaction_duration_seconds` | Histogram | 10.2 | Duration for all transactions (`gitlab_transaction_*` metrics) | `controller`, `action` | | `gitlab_transaction_event_build_found_total` | Counter | 9.4 | Counter for build found for API /jobs/request | | | `gitlab_transaction_event_build_invalid_total` | Counter | 9.4 | Counter for build invalid due to concurrency conflict for API /jobs/request | | | `gitlab_transaction_event_build_not_found_cached_total` | Counter | 9.4 | Counter for cached response of build not found for API /jobs/request | | | `gitlab_transaction_event_build_not_found_total` | Counter | 9.4 | Counter for build not found for API /jobs/request | | | `gitlab_transaction_event_change_default_branch_total` | Counter | 9.4 | Counter when default branch is changed for any repository | | | `gitlab_transaction_event_create_repository_total` | Counter | 9.4 | Counter when any repository is created | | -| `gitlab_transaction_event_etag_caching_cache_hit_total` | Counter | 9.4 | Counter for etag cache hit. | `endpoint` | -| `gitlab_transaction_event_etag_caching_header_missing_total` | Counter | 9.4 | Counter for etag cache miss - header missing | `endpoint` | -| `gitlab_transaction_event_etag_caching_key_not_found_total` | Counter | 9.4 | Counter for etag cache miss - key not found | `endpoint` | -| `gitlab_transaction_event_etag_caching_middleware_used_total` | Counter | 9.4 | Counter for etag middleware accessed | `endpoint` | -| `gitlab_transaction_event_etag_caching_resource_changed_total` | Counter | 9.4 | Counter for etag cache miss - resource changed | `endpoint` | +| `gitlab_transaction_event_etag_caching_cache_hit_total` | Counter | 9.4 | Counter for ETag cache hit. | `endpoint` | +| `gitlab_transaction_event_etag_caching_header_missing_total` | Counter | 9.4 | Counter for ETag cache miss - header missing | `endpoint` | +| `gitlab_transaction_event_etag_caching_key_not_found_total` | Counter | 9.4 | Counter for ETag cache miss - key not found | `endpoint` | +| `gitlab_transaction_event_etag_caching_middleware_used_total` | Counter | 9.4 | Counter for ETag middleware accessed | `endpoint` | +| `gitlab_transaction_event_etag_caching_resource_changed_total` | Counter | 9.4 | Counter for ETag cache miss - resource changed | `endpoint` | | `gitlab_transaction_event_fork_repository_total` | Counter | 9.4 | Counter for repository forks (RepositoryForkWorker). Only incremented when source repository exists | | | `gitlab_transaction_event_import_repository_total` | Counter | 9.4 | Counter for repository imports (RepositoryImportWorker) | | | `gitlab_transaction_event_push_branch_total` | Counter | 9.4 | Counter for all branch pushes | | @@ -92,6 +92,13 @@ The following metrics are available: | `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | `controller`, `action`, `view` | | `http_requests_total` | Counter | 9.4 | Rack request count | `method` | | `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | `method`, `status` | +| `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of sql calls | `controller`, `action` | +| `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write sql calls | `controller`, `action` | +| `gitlab_transaction_db_cached_count_total` | Counter | 13.1 | Counter for total number of cached sql calls | `controller`, `action` | +| `http_redis_requests_duration_seconds` | Histogram | 13.1 | Redis requests duration during web transactions | `controller`, `action` | +| `http_redis_requests_total` | Counter | 13.1 | Redis requests count during web transactions | `controller`, `action` | +| `http_elasticsearch_requests_duration_seconds` **(STARTER)** | Histogram | 13.1 | Elasticsearch requests duration during web transactions | `controller`, `action` | +| `http_elasticsearch_requests_total` **(STARTER)** | Counter | 13.1 | Elasticsearch requests count during web transactions | `controller`, `action` | | `pipelines_created_total` | Counter | 9.4 | Counter of pipelines created | | | `rack_uncaught_errors_total` | Counter | 9.4 | Rack connections handling uncaught errors count | | | `user_session_logins_total` | Counter | 9.4 | Counter of how many users have logged in | | @@ -99,7 +106,7 @@ The following metrics are available: | `failed_login_captcha_total` | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login | | | `successful_login_captcha_total` | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login | | | `auto_devops_pipelines_completed_total` | Counter | 12.7 | Counter of completed Auto DevOps pipelines, labeled by status | | -| `gitlab_metrics_dashboard_processing_time_ms` | Summary | 12.10 | Metrics dashboard processing time in milliseconds | service, stages | +| `gitlab_metrics_dashboard_processing_time_ms` | Summary | 12.10 | Metrics dashboard processing time in milliseconds | service, stages | ## Metrics controlled by a feature flag @@ -119,13 +126,17 @@ configuration option in `gitlab.yml`. These metrics are served from the | Metric | Type | Since | Description | Labels | |:---------------------------------------------- |:------- |:----- |:----------- |:------ | -| `sidekiq_jobs_cpu_seconds` | Histogram | 12.4 | Seconds of cpu time to run Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_jobs_cpu_seconds` | Histogram | 12.4 | Seconds of CPU time to run Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | | `sidekiq_jobs_completion_seconds` | Histogram | 12.2 | Seconds to complete Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | | `sidekiq_jobs_db_seconds` | Histogram | 12.9 | Seconds of DB time to run Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | | `sidekiq_jobs_gitaly_seconds` | Histogram | 12.9 | Seconds of Gitaly time to run Sidekiq job | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_redis_requests_duration_seconds` | Histogram | 13.1 | Duration in seconds that a Sidekiq job spent querying a Redis server | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_elasticsearch_requests_duration_seconds` | Histogram | 13.1 | Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | | `sidekiq_jobs_queue_duration_seconds` | Histogram | 12.5 | Duration in seconds that a Sidekiq job was queued before being executed | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | | `sidekiq_jobs_failed_total` | Counter | 12.2 | Sidekiq jobs failed | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | | `sidekiq_jobs_retried_total` | Counter | 12.2 | Sidekiq jobs retried | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | +| `sidekiq_redis_requests_total` | Counter | 13.1 | Redis requests during a Sidekiq job execution | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | +| `sidekiq_elasticsearch_requests_total` | Counter | 13.1 | Elasticsearch requests during a Sidekiq job execution | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | | `sidekiq_running_jobs` | Gauge | 12.2 | Number of Sidekiq jobs running | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | | `sidekiq_concurrency` | Gauge | 12.5 | Maximum number of Sidekiq jobs | | | `geo_db_replication_lag_seconds` | Gauge | 10.2 | Database replication lag (seconds) | `url` | @@ -172,7 +183,29 @@ The following metrics are available: | Metric | Type | Since | Description | |:--------------------------------- |:--------- |:------------------------------------------------------------- |:-------------------------------------- | -| `db_load_balancing_hosts` | Gauge | [12.3](https://gitlab.com/gitlab-org/gitlab/issues/13630) | Current number of load balancing hosts | +| `db_load_balancing_hosts` | Gauge | [12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/13630) | Current number of load balancing hosts | + +## Connection pool metrics + +These metrics record the status of the database [connection pools](https://api.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html). + +They all have these labels: + +1. `class` - the Ruby class being recorded. + 1. `ActiveRecord::Base` is the main database connection. + 1. `Geo::TrackingBase` is the connection to the Geo tracking database, if + enabled. +1. `host` - the host name used to connect to the database. +1. `port` - the port used to connect to the database. + +| Metric | Type | Since | Description | +|:----------------------------------------------|:------|:------|:--------------------------------------------------| +| `gitlab_database_connection_pool_size` | Gauge | 13.0 | Total connection pool capacity | +| `gitlab_database_connection_pool_connections` | Gauge | 13.0 | Current connections in the pool | +| `gitlab_database_connection_pool_busy` | Gauge | 13.0 | Connections in use where the owner is still alive | +| `gitlab_database_connection_pool_dead` | Gauge | 13.0 | Connections in use where the owner is not alive | +| `gitlab_database_connection_pool_idle` | Gauge | 13.0 | Connections not in use | +| `gitlab_database_connection_pool_waiting` | Gauge | 13.0 | Threads currently waiting on this queue | ## Ruby metrics @@ -205,31 +238,28 @@ Unicorn specific metrics, when Unicorn is used. When Puma is used instead of Unicorn, the following metrics are available: -| Metric | Type | Since | Description | -|:---------------------------------------------- |:------- |:----- |:----------- | -| `puma_workers` | Gauge | 12.0 | Total number of workers | -| `puma_running_workers` | Gauge | 12.0 | Number of booted workers | -| `puma_stale_workers` | Gauge | 12.0 | Number of old workers | -| `puma_running` | Gauge | 12.0 | Number of running threads | -| `puma_queued_connections` | Gauge | 12.0 | Number of connections in that worker's "to do" set waiting for a worker thread | -| `puma_active_connections` | Gauge | 12.0 | Number of threads processing a request | -| `puma_pool_capacity` | Gauge | 12.0 | Number of requests the worker is capable of taking right now | -| `puma_max_threads` | Gauge | 12.0 | Maximum number of worker threads | -| `puma_idle_threads` | Gauge | 12.0 | Number of spawned threads which are not processing a request | -| `puma_killer_terminations_total` | Gauge | 12.0 | Number of workers terminated by PumaWorkerKiller | +| Metric | Type | Since | Description | +|:--------------------------------- |:------- |:----- |:----------- | +| `puma_workers` | Gauge | 12.0 | Total number of workers | +| `puma_running_workers` | Gauge | 12.0 | Number of booted workers | +| `puma_stale_workers` | Gauge | 12.0 | Number of old workers | +| `puma_running` | Gauge | 12.0 | Number of running threads | +| `puma_queued_connections` | Gauge | 12.0 | Number of connections in that worker's "to do" set waiting for a worker thread | +| `puma_active_connections` | Gauge | 12.0 | Number of threads processing a request | +| `puma_pool_capacity` | Gauge | 12.0 | Number of requests the worker is capable of taking right now | +| `puma_max_threads` | Gauge | 12.0 | Maximum number of worker threads | +| `puma_idle_threads` | Gauge | 12.0 | Number of spawned threads which are not processing a request | +| `puma_killer_terminations_total` | Gauge | 12.0 | Number of workers terminated by PumaWorkerKiller | ## Metrics shared directory GitLab's Prometheus client requires a directory to store metrics data shared between multi-process services. Those files are shared among all instances running under Unicorn server. -The directory needs to be accessible to all running Unicorn's processes otherwise -metrics will not function correctly. - -For best performance its advisable that this directory will be located in `tmpfs`. - -Its location is configured using environment variable `prometheus_multiproc_dir`. +The directory must be accessible to all running Unicorn's processes, or +metrics won't function correctly. -If GitLab is installed using Omnibus and `tmpfs` is available then metrics -directory will be automatically configured. +This directory's location is configured using environment variable `prometheus_multiproc_dir`. +For best performance, create this directory in `tmpfs`. -[← Back to the main Prometheus page](index.md) +If GitLab is installed using [Omnibus GitLab](https://docs.gitlab.com/omnibus/) +and `tmpfs` is available, then the metrics directory will be configured for you. diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index cb93aca6e4e..1e233b890a2 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -84,7 +84,7 @@ To change the address/port that Prometheus listens on: 1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect -### Adding custom scrape configs +### Adding custom scrape configurations You can configure additional scrape targets for the Omnibus GitLab-bundled Prometheus by editing `prometheus['scrape_configs']` in `/etc/gitlab/gitlab.rb` diff --git a/doc/administration/monitoring/prometheus/node_exporter.md b/doc/administration/monitoring/prometheus/node_exporter.md index 357303ee4e1..07f6b8b8e1e 100644 --- a/doc/administration/monitoring/prometheus/node_exporter.md +++ b/doc/administration/monitoring/prometheus/node_exporter.md @@ -6,27 +6,24 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Node exporter ->**Note:** -Available since Omnibus GitLab 8.16. For installations from source you'll -have to install and configure it yourself. - -The [node exporter](https://github.com/prometheus/node_exporter) allows you to measure +The [node exporter](https://github.com/prometheus/node_exporter) enables you to measure various machine resources such as memory, disk and CPU utilization. +NOTE: **Note:** +For installations from source you'll have to install and configure it yourself. + To enable the node exporter: -1. [Enable Prometheus](index.md#configuring-prometheus) -1. Edit `/etc/gitlab/gitlab.rb` -1. Add or find and uncomment the following line, making sure it's set to `true`: +1. [Enable Prometheus](index.md#configuring-prometheus). +1. Edit `/etc/gitlab/gitlab.rb`. +1. Add (or find and uncomment) the following line, making sure it's set to `true`: ```ruby node_exporter['enable'] = true ``` -1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect - -Prometheus will now automatically begin collecting performance data from -the node exporter exposed under `localhost:9100`. +1. Save the file, and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. -[← Back to the main Prometheus page](index.md) +Prometheus will now begin collecting performance data from the node exporter +exposed at `localhost:9100`. diff --git a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md index 92ba2d9bb52..62d0bf684b6 100644 --- a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md +++ b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md @@ -6,29 +6,30 @@ info: To determine the technical writer assigned to the Stage/Group associated w # PgBouncer exporter ->**Note:** -Available since [Omnibus GitLab 11.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2493). -For installations from source you'll have to install and configure it yourself. +> Introduced in [Omnibus GitLab 11.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2493). + +The [PgBouncer exporter](https://github.com/prometheus-community/pgbouncer_exporter) enables +you to measure various [PgBouncer](https://www.pgbouncer.org/) metrics. -The [PgBouncer exporter](https://github.com/stanhu/pgbouncer_exporter) allows you to measure various PgBouncer metrics. +NOTE: **Note:** +For installations from source you'll have to install and configure it yourself. To enable the PgBouncer exporter: -1. [Enable Prometheus](index.md#configuring-prometheus) -1. Edit `/etc/gitlab/gitlab.rb` -1. Add or find and uncomment the following line, making sure it's set to `true`: +1. [Enable Prometheus](index.md#configuring-prometheus). +1. Edit `/etc/gitlab/gitlab.rb`. +1. Add (or find and uncomment) the following line, making sure it's set to `true`: ```ruby pgbouncer_exporter['enable'] = true ``` -1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to - take effect. +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. -Prometheus will now automatically begin collecting performance data from -the PgBouncer exporter exposed under `localhost:9188`. +Prometheus will now begin collecting performance data from the PgBouncer exporter +exposed at `localhost:9188`. -The PgBouncer exporter will also be enabled by default if the [`pgbouncer_role`](https://docs.gitlab.com/omnibus/roles/#postgres-roles) +The PgBouncer exporter will also be enabled by default if the +[`pgbouncer_role`](https://docs.gitlab.com/omnibus/roles/#postgresql-roles) role is enabled. - -[← Back to the main Prometheus page](index.md) diff --git a/doc/administration/monitoring/prometheus/postgres_exporter.md b/doc/administration/monitoring/prometheus/postgres_exporter.md index 77ca502b21d..e3fff45fce3 100644 --- a/doc/administration/monitoring/prometheus/postgres_exporter.md +++ b/doc/administration/monitoring/prometheus/postgres_exporter.md @@ -6,12 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # PostgreSQL Server Exporter ->**Note:** -Available since [Omnibus GitLab 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1131). -For installations from source you will have to install and configure it yourself. - The [PostgreSQL Server Exporter](https://github.com/wrouesnel/postgres_exporter) allows you to export various PostgreSQL metrics. +NOTE: **Note:** +For installations from source you will have to install and configure it yourself. + To enable the PostgreSQL Server Exporter: 1. [Enable Prometheus](index.md#configuring-prometheus). @@ -21,10 +20,10 @@ To enable the PostgreSQL Server Exporter: postgres_exporter['enable'] = true ``` -NOTE: **Note:** -If PostgreSQL Server Exporter is configured on a separate node, make sure that the local -address is [listed in `trust_auth_cidr_addresses`](../../high_availability/database.md#network-information) or the -exporter will not be able to connect to the database. + NOTE: **Note:** + If PostgreSQL Server Exporter is configured on a separate node, make sure that the local + address is [listed in `trust_auth_cidr_addresses`](../../postgresql/replication_and_failover.md#network-information) or the + exporter will not be able to connect to the database. 1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -34,31 +33,44 @@ the PostgreSQL Server Exporter exposed under `localhost:9187`. ## Advanced configuration -In most cases, PostgreSQL Server Exporter will work with the defaults and you should not -need to change anything. - -To further customize the PostgreSQL Server Exporter, use the following configuration options: +In most cases, PostgreSQL Server Exporter works with the defaults and you should not +need to change anything. To further customize the PostgreSQL Server Exporter, +use the following configuration options: 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby - postgres_exporter['dbname'] = 'pgbouncer' # The name of the database to connect to. - postgres_exporter['user'] = 'gitlab-psql' # The user to sign in as. - postgres_exporter['password'] = '' # The user's password. - postgres_exporter['host'] = 'localhost' # The host to connect to. Values that start with '/' are for unix domain sockets (default is 'localhost'). - postgres_exporter['port'] = 5432 # The port to bind to (default is '5432'). - postgres_exporter['sslmode'] = 'require' # Whether or not to use SSL. Valid options are: - # 'disable' (no SSL), - # 'require' (always use SSL and skip verification, this is the default value), - # 'verify-ca' (always use SSL and verify that the certificate presented by the server was signed by a trusted CA), - # 'verify-full' (always use SSL and verify that the certification presented by the server was signed by a trusted CA and the server host name matches the one in the certificate). - postgres_exporter['fallback_application_name'] = '' # An application_name to fall back to if one isn't provided. - postgres_exporter['connect_timeout'] = '' # Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. - postgres_exporter['sslcert'] = 'ssl.crt' # Cert file location. The file must contain PEM encoded data. - postgres_exporter['sslkey'] = 'ssl.key' # Key file location. The file must contain PEM encoded data. - postgres_exporter['sslrootcert'] = 'ssl-root.crt' # The location of the root certificate file. The file must contain PEM encoded data. + # The name of the database to connect to. + postgres_exporter['dbname'] = 'pgbouncer' + # The user to sign in as. + postgres_exporter['user'] = 'gitlab-psql' + # The user's password. + postgres_exporter['password'] = '' + # The host to connect to. Values that start with '/' are for unix domain sockets + # (default is 'localhost'). + postgres_exporter['host'] = 'localhost' + # The port to bind to (default is '5432'). + postgres_exporter['port'] = 5432 + # Whether or not to use SSL. Valid options are: + # 'disable' (no SSL), + # 'require' (always use SSL and skip verification, this is the default value), + # 'verify-ca' (always use SSL and verify that the certificate presented by + # the server was signed by a trusted CA), + # 'verify-full' (always use SSL and verify that the certification presented + # by the server was signed by a trusted CA and the server host name matches + # the one in the certificate). + postgres_exporter['sslmode'] = 'require' + # An application_name to fall back to if one isn't provided. + postgres_exporter['fallback_application_name'] = '' + # Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. + postgres_exporter['connect_timeout'] = '' + # Cert file location. The file must contain PEM encoded data. + postgres_exporter['sslcert'] = 'ssl.crt' + # Key file location. The file must contain PEM encoded data. + postgres_exporter['sslkey'] = 'ssl.key' + # The location of the root certificate file. The file must contain PEM encoded data. + postgres_exporter['sslrootcert'] = 'ssl-root.crt' ``` -1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -[← Back to the main Prometheus page](index.md) +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. diff --git a/doc/administration/monitoring/prometheus/redis_exporter.md b/doc/administration/monitoring/prometheus/redis_exporter.md index bef87400f5a..b7c66959349 100644 --- a/doc/administration/monitoring/prometheus/redis_exporter.md +++ b/doc/administration/monitoring/prometheus/redis_exporter.md @@ -6,19 +6,18 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Redis exporter ->**Note:** -Available since [Omnibus GitLab 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1118). -For installations from source you'll have to install and configure it yourself. - -The [Redis exporter](https://github.com/oliver006/redis_exporter) allows you to measure -various [Redis](https://redis.io) metrics. For more information on what's exported, +The [Redis exporter](https://github.com/oliver006/redis_exporter) enables you to measure +various [Redis](https://redis.io) metrics. For more information on what is exported, [read the upstream documentation](https://github.com/oliver006/redis_exporter/blob/master/README.md#whats-exported). +NOTE: **Note:** +For installations from source you'll have to install and configure it yourself. + To enable the Redis exporter: -1. [Enable Prometheus](index.md#configuring-prometheus) -1. Edit `/etc/gitlab/gitlab.rb` -1. Add or find and uncomment the following line, making sure it's set to `true`: +1. [Enable Prometheus](index.md#configuring-prometheus). +1. Edit `/etc/gitlab/gitlab.rb`. +1. Add (or find and uncomment) the following line, making sure it's set to `true`: ```ruby redis_exporter['enable'] = true @@ -27,7 +26,5 @@ To enable the Redis exporter: 1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -Prometheus will now automatically begin collecting performance data from -the Redis exporter exposed under `localhost:9121`. - -[← Back to the main Prometheus page](index.md) +Prometheus will now begin collecting performance data from +the Redis exporter exposed at `localhost:9121`. diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index c4f9b672923..1dea2de73f6 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -66,7 +66,7 @@ One risk of using a single bucket would be that if your organisation decided to migrate GitLab to the Helm deployment in the future. GitLab would run, but the situation with backups might not be realised until the organisation had a critical requirement for the backups to work. -### S3 API compatability issues +### S3 API compatibility issues Not all S3 providers [are fully compatible](../raketasks/backup_restore.md#other-s3-providers) with the Fog library that GitLab uses. Symptoms include: @@ -79,7 +79,7 @@ with the Fog library that GitLab uses. Symptoms include: If you're working to add more GitLab servers for [scaling or fault tolerance](reference_architectures/index.md) and one of your requirements is [GitLab Pages](../user/project/pages/index.md) this currently requires -NFS. There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) +NFS. There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196) to remove this dependency. In the future, GitLab Pages may use [object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/208135). @@ -141,10 +141,88 @@ Using the default GitLab settings, some object storage back-ends such as and [Alibaba](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564) might generate `ETag mismatch` errors. +If you are seeing this ETag mismatch error with Amazon Web Services S3, +it's likely this is due to [encryption settings on your bucket](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html). +See the section on [using Amazon instance profiles](#using-amazon-instance-profiles) on how to fix this issue. + When using GitLab direct upload, the [workaround for MinIO](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564#note_244497658) is to use the `--compat` parameter on the server. -We are working on a fix to GitLab component Workhorse, and also -a workaround, in the mean time, to -[allow ETag verification to be disabled](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18175). +We are working on a fix to the [GitLab Workhorse +component](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/222). + +### Using Amazon instance profiles + +Instead of supplying AWS access and secret keys in object storage +configuration, GitLab can be configured to use IAM roles to set up an +[Amazon instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). +When this is used, GitLab will fetch temporary credentials each time an +S3 bucket is accessed, so no hard-coded values are needed in the +configuration. + +#### Encrypted S3 buckets + +> Introduced in [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466) only for instance profiles. + +When configured to use an instance profile, GitLab Workhorse +will properly upload files to S3 buckets that have [SSE-S3 or SSE-KMS +encryption enabled by default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). +Note that customer master keys (CMKs) and SSE-C encryption are not yet +supported since this requires supplying keys to the GitLab +configuration. + +Without instance profiles enabled (or prior to GitLab 13.1), GitLab +Workhorse will upload files to S3 using pre-signed URLs that do not have +a `Content-MD5` HTTP header computed for them. To ensure data is not +corrupted, Workhorse checks that the MD5 hash of the data sent equals +the ETag header returned from the S3 server. When encryption is enabled, +this is not the case, which causes Workhorse to report an `ETag +mismatch` error during an upload. + +With instance profiles enabled, GitLab Workhorse uses an AWS S3 client +that properly computes and sends the `Content-MD5` header to the server, +which eliminates the need for comparing ETag headers. If the data is +corrupted in transit, the S3 server will reject the file. + +#### IAM Permissions + +To set up an instance profile, create an Amazon Identity Access and +Management (IAM) role with the necessary permissions. The following +example is a role for an S3 bucket named `test-bucket`: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:AbortMultipartUpload", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::test-bucket/*" + } + ] +} +``` + +Associate this role with your GitLab instance, and then configure GitLab +to use it via the `use_iam_profile` configuration option. For example, +when configuring uploads to use object storage, see the `AWS IAM profiles` +section in [S3 compatible connection settings](uploads.md#s3-compatible-connection-settings). + +#### Disabling the feature + +The Workhorse S3 client is only enabled when the `use_iam_profile` +configuration flag is `true`. + +To disable this feature, ask a GitLab administrator with [Rails console access](feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the +following command: + +```ruby +Feature.disable(:use_workhorse_s3_client) +``` diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 8f54b82c325..155680354da 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -106,14 +106,14 @@ you list: ## Queue selector (experimental) -> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/45) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/45) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. CAUTION: **Caution:** As this is marked as **experimental**, it is subject to change at any time, including **breaking backwards compatibility**. This is so that we can react to changes we need for our GitLab.com deployment. We have a tracking issue open to [remove the experimental -designation](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/147) +designation](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/147) from this feature; please comment there if you are interested in using this in your own deployment. @@ -126,6 +126,8 @@ in a more general way using the following components: ### Available attributes +- [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1, `tags`. + From the [list of all available attributes](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml), `experimental_queue_selector` allows selecting of queues by the @@ -144,14 +146,21 @@ following attributes: - `name` - the queue name. The other attributes are typically more useful as they are more general, but this is available in case a particular queue needs to be selected. -- `resource_boundary` - if the worker is bound by `cpu`, `memory`, or +- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or `unknown`. For example, the `project_export` queue is memory bound as it has to load data in memory before saving it for export. +- `tags` - short-lived annotations for queues. These are expected to frequently + change from release to release, and may be removed entirely. `has_external_dependencies` is a boolean attribute: only the exact string `true` is considered true, and everything else is considered false. +`tags` is a set, which means that `=` checks for intersecting sets, and +`!=` checks for disjoint sets. For example, `tags=a,b` selects queues +that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have +neither of those tags. + ### Available operators `experimental_queue_selector` supports the following operators, listed diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 6759c3f265d..9f67c927128 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -1,7 +1,7 @@ # Fast lookup of authorized SSH keys in the database -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1631) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. -> - [Available in](https://gitlab.com/gitlab-org/gitlab/issues/3953) GitLab Community Edition 10.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1631) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.3. +> - [Available in](https://gitlab.com/gitlab-org/gitlab/-/issues/3953) GitLab Community Edition 10.4. NOTE: **Note:** This document describes a drop-in replacement for the `authorized_keys` file. For normal (non-deploy key) users, consider using @@ -125,7 +125,7 @@ the database. The following instructions can be used to build OpenSSH 7.5: ```shell sudo su - cd /tmp - curl --remote-name https://mirrors.evowise.com/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz + curl --remote-name https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz tar xzvf openssh-7.5p1.tar.gz yum install rpm-build gcc make wget openssl-devel krb5-devel pam-devel libX11-devel xmkmf libXt-devel ``` diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index 019909e2e89..c5c5a8b4313 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -14,7 +14,7 @@ I/O. The information on this page can be used for either scenario. ### Benchmarking with `fio` We recommend using -[fio](https://fio.readthedocs.io/en/latest/fio_doc.html) to test I/O +[Fio](https://fio.readthedocs.io/en/latest/fio_doc.html) to test I/O performance. This test should be run both on the NFS server and on the application nodes that talk to the NFS server. diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index af559cf00e9..af28335ef91 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -10,7 +10,8 @@ Unicorn unless explicitly specified not to. ## Why switch to Puma? Puma has a multi-thread architecture which uses less memory than a multi-process -application server like Unicorn. +application server like Unicorn. On GitLab.com, we saw a 40% reduction in memory +consumption. Most Rails applications requests normally include a proportion of I/O wait time. During I/O wait time MRI Ruby will release the GVL (Global VM Lock) to other threads. @@ -18,9 +19,15 @@ Multi-threaded Puma can therefore still serve more requests than a single proces ## Configuring Puma to replace Unicorn -If you are currently running Unicorn and would like to switch to Puma, server configuration -will _not_ carry over automatically. For details on matching Unicorn configuration settings with -the Puma equivalent, where applicable, see [Converting Unicorn settings to Puma](https://docs.gitlab.com/omnibus/settings/puma.html#converting-unicorn-settings-to-puma). +Beginning with GitLab 13.0, Puma is the default application server. We plan to remove support for +Unicorn in GitLab 14.0. + +When switching to Puma, Unicorn server configuration +will _not_ carry over automatically, due to differences between the two application servers. For Omnibus-based +deployments, see [Configuring Puma Settings](https://docs.gitlab.com/omnibus/settings/puma.html#configuring-puma-settings). +For Helm based deployments, see the [Webservice Chart documentation](https://docs.gitlab.com/charts/charts/gitlab/webservice/index.html). + +Additionally we strongly recommend that multi-node deployments [configure their load balancers to utilize the readiness check](../high_availability/load_balancer.md#readiness-check) due to a difference between Unicorn and Puma in how they handle connections during a restart of the service. ## Performance caveat when using Puma with Rugged diff --git a/doc/administration/operations/ssh_certificates.md b/doc/administration/operations/ssh_certificates.md index b652f282b7b..c81eb15955d 100644 --- a/doc/administration/operations/ssh_certificates.md +++ b/doc/administration/operations/ssh_certificates.md @@ -158,7 +158,7 @@ Users can still bypass SSH certificate authentication by manually uploading an SSH public key to their profile, relying on the `~/.ssh/authorized_keys` fallback to authenticate it. There's currently no feature to prevent this, [but there's an open request for -adding it](https://gitlab.com/gitlab-org/gitlab-foss/issues/49218). +adding it](https://gitlab.com/gitlab-org/gitlab/-/issues/23260). Such a restriction can currently be hacked in by e.g. providing a custom `AuthorizedKeysCommand` which checks if the discovered key-ID diff --git a/doc/administration/operations/unicorn.md b/doc/administration/operations/unicorn.md index 50481482f4c..eabf99eb08c 100644 --- a/doc/administration/operations/unicorn.md +++ b/doc/administration/operations/unicorn.md @@ -40,9 +40,9 @@ master process has PID 56227 below. [2015-06-05T10:58:08.708824 #62538] INFO -- : worker=10 ready ``` -### Tunables +### Tunable options -The main tunables for Unicorn are the number of worker processes and the +The main tunable options for Unicorn are the number of worker processes and the request timeout after which the Unicorn master terminates a worker process. See the [Omnibus GitLab Unicorn settings documentation](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/unicorn.md) diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 503e5fb4a2a..8f55345a9a8 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Container Registry administration > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4040) in GitLab 8.8. @@ -92,7 +98,7 @@ auth: ``` CAUTION: **Caution:** -If `auth` is not set up, users will be able to pull docker images without authentication. +If `auth` is not set up, users will be able to pull Docker images without authentication. ## Container Registry domain configuration @@ -116,7 +122,7 @@ expose the Registry on a port so that you can reuse the existing GitLab TLS certificate. Assuming that the GitLab domain is `https://gitlab.example.com` and the port the -Registry is exposed to the outside world is `4567`, here is what you need to set +Registry is exposed to the outside world is `5050`, here is what you need to set in `gitlab.rb` or `gitlab.yml` if you are using Omnibus GitLab or installed GitLab from source respectively. @@ -130,7 +136,7 @@ otherwise you will run into conflicts. path to the existing TLS certificate and key used by GitLab: ```ruby - registry_external_url 'https://gitlab.example.com:4567' + registry_external_url 'https://gitlab.example.com:5050' ``` Note how the `registry_external_url` is listening on HTTPS under the @@ -151,7 +157,7 @@ otherwise you will run into conflicts. 1. Validate using: ```shell - openssl s_client -showcerts -servername gitlab.example.com -connect gitlab.example.com:443 > cacert.pem + openssl s_client -showcerts -servername gitlab.example.com -connect gitlab.example.com:5050 > cacert.pem ``` NOTE: **Note:** @@ -166,7 +172,7 @@ If your certificate provider provides the CA Bundle certificates, append them to registry: enabled: true host: gitlab.example.com - port: 4567 + port: 5050 ``` 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -176,7 +182,7 @@ Users should now be able to login to the Container Registry with their GitLab credentials using: ```shell -docker login gitlab.example.com:4567 +docker login gitlab.example.com:5050 ``` ### Configure Container Registry under its own domain @@ -370,7 +376,7 @@ The different supported drivers are: | swift | OpenStack Swift Object Storage | | oss | Aliyun OSS | -Read more about the individual driver's config options in the +Read more about the individual driver's configuration options in the [Docker Registry docs](https://docs.docker.com/registry/configuration/#storage). [Read more about using object storage with GitLab](../object_storage.md). @@ -407,8 +413,8 @@ NOTE: **Note:** **Installations from source** -Configuring the storage driver is done in your registry config YML file created -when you [deployed your docker registry](https://docs.docker.com/registry/deploying/). +Configuring the storage driver is done in your registry configuration YML file created +when you [deployed your Docker registry](https://docs.docker.com/registry/deploying/). `s3` storage driver example: @@ -429,6 +435,16 @@ storage: NOTE: **Note:** `your-s3-bucket` should only be the name of a bucket that exists, and can't include subdirectories. +**Migrate without downtime** + +To migrate the data to AWS S3 without downtime: + +1. To reduce the amount of data to be migrated, run the [garbage collection tool without downtime](#performing-garbage-collection-without-downtime). Part of this process sets the registry to `read-only`. +1. Copy the data to your AWS S3 bucket, for example with [AWS CLI's `cp`](https://docs.aws.amazon.com/cli/latest/reference/s3/cp.html) command. +1. Configure your registry to use the S3 bucket for storage. +1. Put the registry back to `read-write`. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + ### Disable redirect for storage driver By default, users accessing a registry configured with a remote backend are redirected to the default backend for the storage driver. For example, registries can be configured using the `s3` storage driver, which redirects requests to a remote S3 bucket to alleviate load on the GitLab server. @@ -604,7 +620,7 @@ You can use GitLab as an auth endpoint with an external container registry. You can configure the Container Registry to send webhook notifications in response to events happening within the registry. -Read more about the Container Registry notifications config options in the +Read more about the Container Registry notifications configuration options in the [Docker Registry notifications documentation](https://docs.docker.com/registry/notifications/). NOTE: **Note:** @@ -635,8 +651,8 @@ To configure a notification endpoint in Omnibus: **Installations from source** -Configuring the notification endpoint is done in your registry config YML file created -when you [deployed your docker registry](https://docs.docker.com/registry/deploying/). +Configuring the notification endpoint is done in your registry configuration YML file created +when you [deployed your Docker registry](https://docs.docker.com/registry/deploying/). Example: @@ -704,6 +720,8 @@ no longer directly accessible via the `:latest` tag. ### Recycling unused tags +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/987) in Omnibus GitLab 8.12. + There are a couple of considerations you need to note before running the built-in command: @@ -759,6 +777,8 @@ that you have backed up all registry data. ### Performing garbage collection without downtime +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/764) in GitLab 8.8. + You can perform a garbage collection without stopping the Container Registry by setting it into a read-only mode and by not using the built-in command. During this time, you will be able to pull from the Container Registry, but you will not be able to @@ -873,9 +893,9 @@ thus the error above. While GitLab doesn't support using self-signed certificates with Container Registry out of the box, it is possible to make it work by -[instructing the docker-daemon to trust the self-signed certificates](https://docs.docker.com/registry/insecure/#use-self-signed-certificates), -mounting the docker-daemon and setting `privileged = false` in the Runner's -`config.toml`. Setting `privileged = true` takes precedence over the docker-daemon: +[instructing the Docker daemon to trust the self-signed certificates](https://docs.docker.com/registry/insecure/#use-self-signed-certificates), +mounting the Docker daemon and setting `privileged = false` in the Runner's +`config.toml`. Setting `privileged = true` takes precedence over the Docker daemon: ```toml [runners.docker] @@ -884,7 +904,7 @@ mounting the docker-daemon and setting `privileged = false` in the Runner's volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"] ``` -Additional information about this: [issue 18239](https://gitlab.com/gitlab-org/gitlab-foss/issues/18239). +Additional information about this: [issue 18239](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/18239). ### `unauthorized: authentication required` when pushing large images @@ -955,7 +975,7 @@ Start with a value between `25000000` (25MB) and `50000000` (50MB). ### Supporting older Docker clients -As of GitLab 11.9, we began shipping version 2.7.1 of the Docker container registry, which disables the schema1 manifest by default. If you are still using older Docker clients (1.9 or older), you may experience an error pushing images. See [omnibus-4145](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4145) for more details. +As of GitLab 11.9, we began shipping version 2.7.1 of the Docker container registry, which disables the schema1 manifest by default. If you are still using older Docker clients (1.9 or older), you may experience an error pushing images. See [omnibus-4145](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4145) for more details. You can add a configuration option for backwards compatibility. @@ -1002,7 +1022,7 @@ there is likely an issue with the headers forwarded to the registry by NGINX. Th NGINX configurations should handle this, but it might occur in custom setups where the SSL is offloaded to a third party reverse proxy. -This problem was discussed in a [docker project issue](https://github.com/docker/distribution/issues/970) +This problem was discussed in a [Docker project issue](https://github.com/docker/distribution/issues/970) and a simple solution would be to enable relative URLs in the Registry. **For Omnibus installations** @@ -1062,7 +1082,7 @@ A user attempted to enable an S3-backed Registry. The `docker login` step went fine. However, when pushing an image, the output showed: ```plaintext -The push refers to a repository [s3-testing.myregistry.com:4567/root/docker-test/docker-image] +The push refers to a repository [s3-testing.myregistry.com:5050/root/docker-test/docker-image] dc5e59c14160: Pushing [==================================================>] 14.85 kB 03c20c1a019a: Pushing [==================================================>] 2.048 kB a08f14ef632e: Pushing [==================================================>] 2.048 kB @@ -1154,8 +1174,8 @@ Now that we have mitmproxy and Docker running, we can attempt to login and push a container image. You may need to run as root to do this. For example: ```shell -docker login s3-testing.myregistry.com:4567 -docker push s3-testing.myregistry.com:4567/root/docker-test/docker-image +docker login s3-testing.myregistry.com:5050 +docker push s3-testing.myregistry.com:5050/root/docker-test/docker-image ``` In the example above, we see the following trace on the mitmproxy window: diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index ec2020c26de..565a4521c2a 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -1,6 +1,12 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Dependency Proxy administration **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11. GitLab can be utilized as a dependency proxy for a variety of common package managers. diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index f826741d66f..5088dd86a86 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Package Registry administration **(PREMIUM ONLY)** GitLab Packages allows organizations to utilize GitLab as a private repository @@ -9,10 +15,12 @@ The Packages feature allows GitLab to act as a repository for the following: | Software repository | Description | Available in GitLab version | | ------------------- | ----------- | --------------------------- | | [PyPi Repository](../../user/packages/pypi_repository/index.md) | The GitLab PyPi Repository enables every project in GitLab to have its own space to store [PyPi](https://pypi.org/) packages. | 12.10+ | +| [Composer Repository](../../user/packages/composer_repository/index.md) | The GitLab Composer Repository enables every project in GitLab to have its own space to store [Composer](https://getcomposer.org/) packages. | 13.1+ | | [NuGet Repository](../../user/packages/nuget_repository/index.md) | The GitLab NuGet Repository enables every project in GitLab to have its own space to store [NuGet](https://www.nuget.org/) packages. | 12.8+ | | [Conan Repository](../../user/packages/conan_repository/index.md) | The GitLab Conan Repository enables every project in GitLab to have its own space to store [Conan](https://conan.io/) packages. | 12.4+ | | [Maven Repository](../../user/packages/maven_repository/index.md) | The GitLab Maven Repository enables every project in GitLab to have its own space to store [Maven](https://maven.apache.org/) packages. | 11.3+ | | [NPM Registry](../../user/packages/npm_registry/index.md) | The GitLab NPM Registry enables every project in GitLab to have its own space to store [NPM](https://www.npmjs.com/) packages. | 11.7+ | +| [Go Proxy](../../user/packages/go_proxy/index.md) | The Go proxy for GitLab enables every project in GitLab to be fetched with the [Go proxy protocol](https://proxy.golang.org/). | 13.1+ | Don't you see your package management system supported yet? Please consider contributing diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 21d13be47bd..a7a3a86de8e 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers description: 'Learn how to administer GitLab Pages.' --- @@ -6,9 +9,9 @@ description: 'Learn how to administer GitLab Pages.' > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80) in GitLab EE 8.3. > - Custom CNAMEs with TLS support were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173) in GitLab EE 8.5. -> - GitLab Pages [was ported](https://gitlab.com/gitlab-org/gitlab-foss/issues/14605) to Community Edition in GitLab 8.17. +> - GitLab Pages [was ported](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14605) to Community Edition in GitLab 8.17. > - Support for subgroup project's websites was -> [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/30548) in GitLab 11.8. +> [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30548) in GitLab 11.8. GitLab Pages allows for hosting of static sites. It must be configured by an administrator. Separate [user documentation](../../user/project/pages/index.md) is available. @@ -75,7 +78,7 @@ among other things. Follow [these instructions](https://publicsuffix.org/submit/) to submit your GitLab Pages subdomain. For instance, if your domain is `example.io`, you should request that `example.io` is added to the Public Suffix List. GitLab.com -added `gitlab.io` [in 2016](https://gitlab.com/gitlab-com/infrastructure/issues/230). +added `gitlab.io` [in 2016](https://gitlab.com/gitlab-com/infrastructure/-/issues/230). ### DNS configuration @@ -98,7 +101,8 @@ You should not use the GitLab domain to serve user pages. For more information s ## Configuration Depending on your needs, you can set up GitLab Pages in 4 different ways. -The following options are listed from the easiest setup to the most + +The following examples are listed from the easiest setup to the most advanced one. The absolute minimum requirement is to set up the wildcard DNS since that is needed in all configurations. @@ -175,6 +179,64 @@ NOTE: **Note:** `inplace_chroot` option might not work with the other features, such as [Pages Access Control](#access-control). The [GitLab Pages README](https://gitlab.com/gitlab-org/gitlab-pages#caveats) has more information about caveats and workarounds. +### Global settings + +Below is a table of all configuration settings known to Pages in Omnibus GitLab, +and what they do. These options can be adjusted in `/etc/gitlab/gitlab.rb`, +and will take effect after you [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +Most of these settings don't need to be configured manually unless you need more granular +control over how the Pages daemon runs and serves content in your environment. + +| Setting | Description | +| ------- | ----------- | +| `pages_external_url` | The URL where GitLab Pages is accessible, including protocol (HTTP / HTTPS). If `https://` is used, you must also set `gitlab_pages['ssl_certificate']` and `gitlab_pages['ssl_certificate_key']`. +| **gitlab_pages[]** | | +| `access_control` | Whether to enable [access control](index.md#access-control). +| `api_secret_key` | Full path to file with secret key used to authenticate with the GitLab API. Auto-generated when left unset. +| `artifacts_server` | Enable viewing [artifacts](../job_artifacts.md) in GitLab Pages. +| `artifacts_server_timeout` | Timeout (in seconds) for a proxied request to the artifacts server. +| `artifacts_server_url` | API URL to proxy artifact requests to. Defaults to GitLab `external URL` + `/api/v4`, for example `https://gitlab.com/api/v4`. +| `auth_redirect_uri` | Callback URL for authenticating with GitLab. Defaults to project's subdomain of `pages_external_url` + `/auth`. +| `auth_secret` | Secret key for signing authentication requests. Leave blank to pull automatically from GitLab during OAuth registration. +| `dir` | Working directory for config and secrets files. +| `enable` | Enable or disable GitLab Pages on the current system. +| `external_http` | Configure Pages to bind to one or more secondary IP addresses, serving HTTP requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_http`. +| `external_https` | Configure Pages to bind to one or more secondary IP addresses, serving HTTPS requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_https`. +| `gitlab_client_http_timeout` | GitLab API HTTP client connection timeout in seconds (default: 10s). +| `gitlab_client_jwt_expiry` | JWT Token expiry time in seconds (default: 30s). +| `gitlab_id` | The OAuth application public ID. Leave blank to automatically fill when Pages authenticates with GitLab. +| `gitlab_secret` | The OAuth application secret. Leave blank to automatically fill when Pages authenticates with GitLab. +| `gitlab_server` | Server to use for authentication when access control is enabled; defaults to GitLab `external_url`. +| `headers` | Specify any additional http headers that should be sent to the client with each response. +| `inplace_chroot` | On [systems that don't support bind-mounts](index.md#additional-configuration-for-docker-container), this instructs GitLab Pages to chroot into its `pages_path` directory. Some caveats exist when using inplace chroot; refer to the GitLab Pages [README](https://gitlab.com/gitlab-org/gitlab-pages/blob/master/README.md#caveats) for more information. +| `insecure_ciphers` | Use default list of cipher suites, may contain insecure ones like 3DES and RC4. +| `internal_gitlab_server` | Internal GitLab server address used exclusively for API requests. Useful if you want to send that traffic over an internal load balancer. Defaults to GitLab `external_url`. +| `listen_proxy` | The addresses to listen on for reverse-proxy requests. Pages will bind to these addresses' network socket and receives incoming requests from it. Sets the value of `proxy_pass` in `$nginx-dir/conf/gitlab-pages.conf`. +| `log_directory` | Absolute path to a log directory. +| `log_format` | The log output format: 'text' or 'json'. +| `log_verbose` | Verbose logging, true/false. +| `max_connections` | Limit on the number of concurrent connections to the HTTP, HTTPS or proxy listeners. +| `metrics_address` | The address to listen on for metrics requests. +| `redirect_http` | Redirect pages from HTTP to HTTPS, true/false. +| `sentry_dsn` | The address for sending Sentry crash reporting to. +| `sentry_enabled` | Enable reporting and logging with Sentry, true/false. +| `sentry_environment` | The environment for Sentry crash reporting. +| `status_uri` | The URL path for a status page, for example, `/@status`. +| `tls_max_version` | Specifies the maximum SSL/TLS version ("ssl3", "tls1.0", "tls1.1" or "tls1.2"). +| `tls_min_version` | Specifies the minimum SSL/TLS version ("ssl3", "tls1.0", "tls1.1" or "tls1.2"). +| `use_http2` | Enable HTTP2 support. +| **gitlab_pages['env'][]** | | +| `http_proxy` | Configure GitLab Pages to use an HTTP Proxy to mediate traffic between Pages and GitLab. Sets an environment variable `http_proxy` when starting Pages daemon. +| **gitlab_rails[]** | | +| `pages_domain_verification_cron_worker` | Schedule for verifying custom GitLab Pages domains. +| `pages_domain_ssl_renewal_cron_worker` | Schedule for obtaining and renewing SSL certificates through Let's Encrypt for GitLab Pages domains. +| `pages_domain_removal_cron_worker` | Schedule for removing unverified custom GitLab Pages domains. +| `pages_path` | The directory on disk where pages are stored, defaults to `GITLAB-RAILS/shared/pages`. +| **pages_nginx[]** | | +| `enable` | Include a virtual host `server{}` block for Pages inside NGINX. Needed for NGINX to proxy traffic back to the Pages daemon. Set to `false` if the Pages daemon should directly receive all requests, for example, when using [custom domains](index.md#custom-domains). + +--- + ## Advanced configuration In addition to the wildcard domains, you can also have the option to configure @@ -261,7 +323,7 @@ This setting is enabled by default. ### Let's Encrypt integration -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28996) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28996) in GitLab 12.1. [GitLab Pages' Let's Encrypt integration](../../user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md) allows users to add Let's Encrypt SSL certificates for GitLab Pages @@ -278,7 +340,7 @@ To enable it, you'll need to: ### Access control -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/33422) in GitLab 11.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/33422) in GitLab 11.5. GitLab Pages access control can be configured per-project, and allows access to a Pages site to be controlled based on a user's membership to that project. @@ -305,9 +367,13 @@ Pages access control is disabled by default. To enable it: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Users can now configure it in their [projects' settings](../../user/project/pages/pages_access_control.md). +NOTE: **Important:** +For multi-node setups, in order for this setting to be effective, it has to be applied +to all the App nodes as well as the Sidekiq nodes. + #### Disabling public access to all Pages websites -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32095) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32095) in GitLab 12.7. You can enforce [Access Control](#access-control) for all GitLab Pages websites hosted on your GitLab instance. By doing so, only logged-in users will have access to them. @@ -324,7 +390,7 @@ To do that: CAUTION: **Warning:** This action will not make all currently public web-sites private until they redeployed. This issue among others will be resolved by -[changing GitLab Pages configuration mechanism](https://gitlab.com/gitlab-org/gitlab-pages/issues/282). +[changing GitLab Pages configuration mechanism](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/282). ### Running behind a proxy @@ -335,7 +401,7 @@ pages: 1. Configure in `/etc/gitlab/gitlab.rb`: ```ruby - gitlab_pages['http_proxy'] = 'http://example:8080' + gitlab_pages['env']['http_proxy'] = 'http://example:8080' ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -353,7 +419,7 @@ For installation from source this can be fixed by installing the custom Certific Authority (CA) in the system certificate store. For Omnibus, normally this would be fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) -but a [bug](https://gitlab.com/gitlab-org/gitlab/issues/25411) is currently preventing +but a [bug](https://gitlab.com/gitlab-org/gitlab/-/issues/25411) is currently preventing that method from working. Use the following workaround: 1. Append your GitLab server TLS/SSL certificate to `/opt/gitlab/embedded/ssl/certs/cacert.pem` where `gitlab-domain-example.com` is your GitLab application URL @@ -435,7 +501,7 @@ The default is 100MB. ### Override maximum pages size per project or group **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/16610) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16610) in GitLab 12.7. To override the global maximum pages size for a specific project: diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index 1bb3b86b419..d5b49bdf839 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Pages administration for source installations >**Note:** @@ -266,7 +272,7 @@ world. Custom domains are supported, but no TLS. sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages.conf ``` -1. Edit all GitLab related configs in `/etc/nginx/site-available/` and replace +1. Edit all GitLab related configurations in `/etc/nginx/site-available/` and replace `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab listens to. 1. Restart NGINX @@ -335,7 +341,7 @@ world. Custom domains and TLS are supported. sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages-ssl.conf ``` -1. Edit all GitLab related configs in `/etc/nginx/site-available/` and replace +1. Edit all GitLab related configurations in `/etc/nginx/site-available/` and replace `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab listens to. 1. Restart NGINX @@ -346,7 +352,7 @@ world. Custom domains and TLS are supported. >**Note:** The following information applies only for installations from source. -Be extra careful when setting up the domain name in the NGINX config. You must +Be extra careful when setting up the domain name in the NGINX configuration. You must not remove the backslashes. If your GitLab Pages domain is `example.io`, replace: @@ -370,7 +376,7 @@ server_name ~^.*\.pages\.example\.io$; ## Access control -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/33422) in GitLab 11.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/33422) in GitLab 11.5. GitLab Pages access control can be configured per-project, and allows access to a Pages site to be controlled based on a user's membership to that project. diff --git a/doc/administration/postgresql/external.md b/doc/administration/postgresql/external.md new file mode 100644 index 00000000000..6e2bbc0aae1 --- /dev/null +++ b/doc/administration/postgresql/external.md @@ -0,0 +1,41 @@ +# Configure GitLab using an external PostgreSQL service + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed Relational +Database Service (RDS) that runs PostgreSQL. + +Alternatively, you may opt to manage your own PostgreSQL instance or cluster +separate from the Omnibus GitLab package. + +If you use a cloud-managed service, or provide your own PostgreSQL instance: + +1. Set up PostgreSQL according to the + [database requirements document](../../install/requirements.md#database). +1. Set up a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. If you are using a cloud-managed service, you may need to grant additional + roles to your `gitlab` user: + - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. + - Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. + +1. Configure the GitLab application servers with the appropriate connection details + for your external PostgreSQL service in your `/etc/gitlab/gitlab.rb` file: + + ```ruby + # Disable the bundled Omnibus provided PostgreSQL + postgresql['enable'] = false + + # PostgreSQL connection details + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server + gitlab_rails['db_password'] = 'DB password' + ``` + + For more information on GitLab HA setups, refer to [configuring GitLab for HA](../high_availability/gitlab.md). + +1. Reconfigure for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` diff --git a/doc/administration/high_availability/img/pg_ha_architecture.png b/doc/administration/postgresql/img/pg_ha_architecture.png Binary files differindex ef870f652ae..ef870f652ae 100644 --- a/doc/administration/high_availability/img/pg_ha_architecture.png +++ b/doc/administration/postgresql/img/pg_ha_architecture.png diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md new file mode 100644 index 00000000000..aa95b983d20 --- /dev/null +++ b/doc/administration/postgresql/replication_and_failover.md @@ -0,0 +1,1129 @@ +# PostgreSQL replication and failover with Omnibus GitLab **(PREMIUM ONLY)** + +> Important notes: +> +> - This document will focus only on configuration supported with [GitLab Premium](https://about.gitlab.com/pricing/), using the Omnibus GitLab package. +> - If you are a Community Edition or Starter user, consider using a cloud hosted solution. +> - This document will not cover installations from source. +> +> - If a setup with replication and failover is not what you were looking for, see the [database configuration document](https://docs.gitlab.com/omnibus/settings/database.html) +> for the Omnibus GitLab packages. +> +> Please read this document fully before attempting to configure PostgreSQL with +> replication and failover for GitLab. + +## Architecture + +The Omnibus GitLab recommended configuration for a PostgreSQL cluster with +replication and failover requires: + +- A minimum of three database nodes. +- A minimum of three `Consul` server nodes. +- A minimum of one `pgbouncer` service node, but it's recommended to have one + per database node. + - An internal load balancer (TCP) is required when there is more than one + `pgbouncer` service node. + + + +You also need to take into consideration the underlying network topology, making +sure you have redundant connectivity between all Database and GitLab instances +to avoid the network becoming a single point of failure. + +### Database node + +Each database node runs three services: + +`PostgreSQL` - The database itself. + +`repmgrd` - Communicates with other repmgrd services in the cluster and handles +failover when issues with the master server occurs. The failover procedure +consists of: + +- Selecting a new master for the cluster. +- Promoting the new node to master. +- Instructing remaining servers to follow the new master node. +- The old master node is automatically evicted from the cluster and should be + rejoined manually once recovered. + +`Consul` agent - Monitors the status of each node in the database cluster and +tracks its health in a service definition on the Consul cluster. + +### Consul server node + +The Consul server node runs the Consul server service. + +### PgBouncer node + +Each PgBouncer node runs two services: + +`PgBouncer` - The database connection pooler itself. + +`Consul` agent - Watches the status of the PostgreSQL service definition on the +Consul cluster. If that status changes, Consul runs a script which updates the +PgBouncer configuration to point to the new PostgreSQL master node and reloads +the PgBouncer service. + +### Connection flow + +Each service in the package comes with a set of [default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#ports). You may need to make specific firewall rules for the connections listed below: + +- Application servers connect to either PgBouncer directly via its [default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#pgbouncer) or via a configured Internal Load Balancer (TCP) that serves multiple PgBouncers. +- PgBouncer connects to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) +- Repmgr connects to the database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) +- PostgreSQL secondaries connect to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) +- Consul servers and agents connect to each others [Consul default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#consul) + +## Setting it up + +### Required information + +Before proceeding with configuration, you will need to collect all the necessary +information. + +#### Network information + +PostgreSQL does not listen on any network interface by default. It needs to know +which IP address to listen on in order to be accessible to other services. +Similarly, PostgreSQL access is controlled based on the network source. + +This is why you will need: + +- IP address of each nodes network interface. This can be set to `0.0.0.0` to + listen on all interfaces. It cannot be set to the loopback address `127.0.0.1`. +- Network Address. This can be in subnet (i.e. `192.168.0.0/255.255.255.0`) + or CIDR (i.e. `192.168.0.0/24`) form. + +#### Consul information + +When using default setup, minimum configuration requires: + +- `CONSUL_USERNAME`. Defaults to `gitlab-consul` +- `CONSUL_DATABASE_PASSWORD`. Password for the database user. +- `CONSUL_PASSWORD_HASH`. This is a hash generated out of Consul username/password pair. + Can be generated with: + + ```shell + sudo gitlab-ctl pg-password-md5 CONSUL_USERNAME + ``` + +- `CONSUL_SERVER_NODES`. The IP addresses or DNS records of the Consul server nodes. + +Few notes on the service itself: + +- The service runs under a system account, by default `gitlab-consul`. + - If you are using a different username, you will have to specify it. We + will refer to it with `CONSUL_USERNAME`, +- There will be a database user created with read only access to the repmgr + database +- Passwords will be stored in the following locations: + - `/etc/gitlab/gitlab.rb`: hashed + - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed + - `/var/opt/gitlab/consul/.pgpass`: plaintext + +#### PostgreSQL information + +When configuring PostgreSQL, we will set `max_wal_senders` to one more than +the number of database nodes in the cluster. +This is used to prevent replication from using up all of the +available database connections. + +In this document we are assuming 3 database nodes, which makes this configuration: + +```ruby +postgresql['max_wal_senders'] = 4 +``` + +As previously mentioned, you'll have to prepare the network subnets that will +be allowed to authenticate with the database. +You'll also need to supply the IP addresses or DNS records of Consul +server nodes. + +We will need the following password information for the application's database user: + +- `POSTGRESQL_USERNAME`. Defaults to `gitlab` +- `POSTGRESQL_USER_PASSWORD`. The password for the database user +- `POSTGRESQL_PASSWORD_HASH`. This is a hash generated out of the username/password pair. + Can be generated with: + + ```shell + sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME + ``` + +#### PgBouncer information + +When using default setup, minimum configuration requires: + +- `PGBOUNCER_USERNAME`. Defaults to `pgbouncer` +- `PGBOUNCER_PASSWORD`. This is a password for PgBouncer service. +- `PGBOUNCER_PASSWORD_HASH`. This is a hash generated out of PgBouncer username/password pair. + Can be generated with: + + ```shell + sudo gitlab-ctl pg-password-md5 PGBOUNCER_USERNAME + ``` + +- `PGBOUNCER_NODE`, is the IP address or a FQDN of the node running PgBouncer. + +Few notes on the service itself: + +- The service runs as the same system account as the database + - In the package, this is by default `gitlab-psql` +- If you use a non-default user account for PgBouncer service (by default `pgbouncer`), you will have to specify this username. We will refer to this requirement with `PGBOUNCER_USERNAME`. +- The service will have a regular database user account generated for it + - This defaults to `repmgr` +- Passwords will be stored in the following locations: + - `/etc/gitlab/gitlab.rb`: hashed, and in plain text + - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed + +#### Repmgr information + +When using default setup, you will only have to prepare the network subnets that will +be allowed to authenticate with the service. + +Few notes on the service itself: + +- The service runs under the same system account as the database + - In the package, this is by default `gitlab-psql` +- The service will have a superuser database user account generated for it + - This defaults to `gitlab_repmgr` + +### Installing Omnibus GitLab + +First, make sure to [download/install](https://about.gitlab.com/install/) +Omnibus GitLab **on each node**. + +Make sure you install the necessary dependencies from step 1, +add GitLab package repository from step 2. +When installing the GitLab package, do not supply `EXTERNAL_URL` value. + +### Configuring the Database nodes + +1. Make sure to [configure the Consul nodes](../high_availability/consul.md). +1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information), [`POSTGRESQL_PASSWORD_HASH`](#postgresql-information), the [number of db nodes](#postgresql-information), and the [network address](#network-information) before executing the next step. + +1. On the master database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the Consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + # Replace X with value of number of db nodes + 1 + postgresql['max_wal_senders'] = X + postgresql['max_replication_slots'] = X + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) + + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + # + # END user configuration + ``` + + > `postgres_role` was introduced with GitLab 10.3 + +1. On secondary nodes, add all the configuration specified above for primary node + to `/etc/gitlab/gitlab.rb`. In addition, append the following configuration + to inform `gitlab-ctl` that they are standby nodes initially and it need not + attempt to register them as primary node + + ```ruby + # Specify if a node should attempt to be master on initialization + repmgr['master_on_initialization'] = false + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. [Enable Monitoring](#enable-monitoring) + +> Please note: +> +> - If you want your database to listen on a specific interface, change the configuration: +> `postgresql['listen_address'] = '0.0.0.0'`. +> - If your PgBouncer service runs under a different user account, +> you also need to specify: `postgresql['pgbouncer_user'] = PGBOUNCER_USERNAME` in +> your configuration. + +#### Enable Monitoring + +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. + +If you enable Monitoring, it must be enabled on **all** database servers. + +1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: + + ```ruby + # Enable service discovery for Prometheus + consul['monitoring_service_discovery'] = true + + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + ``` + +1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. + +#### Database nodes post-configuration + +##### Primary node + +Select one node as a primary node. + +1. Open a database prompt: + + ```shell + gitlab-psql -d gitlabhq_production + ``` + +1. Enable the `pg_trgm` extension: + + ```shell + CREATE EXTENSION pg_trgm; + ``` + +1. Exit the database prompt by typing `\q` and Enter. + +1. Verify the cluster is initialized with one node: + + ```shell + gitlab-ctl repmgr cluster show + ``` + + The output should be similar to the following: + + ```plaintext + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will + refer to the hostname in the next section as `MASTER_NODE_NAME`. If the value + is not an IP address, it will need to be a resolvable name (via DNS or + `/etc/hosts`) + +##### Secondary nodes + +1. Set up the repmgr standby: + + ```shell + gitlab-ctl repmgr standby setup MASTER_NODE_NAME + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + # gitlab-ctl repmgr standby setup MASTER_NODE_NAME + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` + +1. Verify the node now appears in the cluster: + + ```shell + gitlab-ctl repmgr cluster show + ``` + + The output should be similar to the following: + + ```plaintext + Role | Name | Upstream | Connection String + ----------+---------|-----------|------------------------------------------------ + * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +Repeat the above steps on all secondary nodes. + +#### Database checkpoint + +Before moving on, make sure the databases are configured correctly. Run the +following command on the **primary** node to verify that replication is working +properly: + +```shell +gitlab-ctl repmgr cluster show +``` + +The output should be similar to: + +```plaintext +Role | Name | Upstream | Connection String +----------+--------------|--------------|-------------------------------------------------------------------- +* master | MASTER | | host=MASTER port=5432 user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=STANDBY port=5432 user=gitlab_repmgr dbname=gitlab_repmgr +``` + +If the 'Role' column for any node says "FAILED", check the +[Troubleshooting section](#troubleshooting) before proceeding. + +Also, check that the check master command works successfully on each node: + +```shell +su - gitlab-consul +gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' +``` + +This command relies on exit codes to tell Consul whether a particular node is a master +or secondary. The most important thing here is that this command does not produce errors. +If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. +Check the [Troubleshooting section](#troubleshooting) before proceeding. + +### Configuring the PgBouncer node + +1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`CONSUL_PASSWORD_HASH`](#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information) before executing the next step. + +1. One each node, edit the `/etc/gitlab/gitlab.rb` config file and replace values noted in the `# START user configuration` section as below: + + ```ruby + # Disable all components except PgBouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure PgBouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # Replace CONSUL_PASSWORD_HASH with with a generated md5 value + # Replace PGBOUNCER_PASSWORD_HASH with with a generated md5 value + pgbouncer['users'] = { + 'gitlab-consul': { + password: 'CONSUL_PASSWORD_HASH' + }, + 'pgbouncer': { + password: 'PGBOUNCER_PASSWORD_HASH' + } + } + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + # + # END user configuration + ``` + + NOTE: **Note:** + `pgbouncer_role` was introduced with GitLab 10.3. + +1. Run `gitlab-ctl reconfigure` + +1. Create a `.pgpass` file so Consul is able to + reload PgBouncer. Enter the `PGBOUNCER_PASSWORD` twice when asked: + + ```shell + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +1. [Enable monitoring](../high_availability/pgbouncer.md#enable-monitoring) + +#### PgBouncer Checkpoint + +1. Ensure each node is talking to the current master: + + ```shell + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + + If there is an error `psql: ERROR: Auth failed` after typing in the + password, ensure you previously generated the MD5 password hashes with the correct + format. The correct format is to concatenate the password and the username: + `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text + needed to generate an MD5 password hash for the `pgbouncer` user. + +1. Once the console prompt is available, run the following queries: + + ```shell + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ```plaintext + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` + +#### Configure the internal load balancer + +If you're running more than one PgBouncer node as recommended, then at this time you'll need to set up a TCP internal load balancer to serve each correctly. This can be done with any reputable TCP load balancer. + +As an example here's how you could do it with [HAProxy](https://www.haproxy.org/): + +```plaintext +global + log /dev/log local0 + log localhost local1 notice + log stdout format raw local0 + +defaults + log global + default-server inter 10s fall 3 rise 2 + balance leastconn + +frontend internal-pgbouncer-tcp-in + bind *:6432 + mode tcp + option tcplog + + default_backend pgbouncer + +backend pgbouncer + mode tcp + option tcp-check + + server pgbouncer1 <ip>:6432 check + server pgbouncer2 <ip>:6432 check + server pgbouncer3 <ip>:6432 check +``` + +Refer to your preferred Load Balancer's documentation for further guidance. + +### Configuring the Application nodes + +These will be the nodes running the `gitlab-rails` service. You may have other +attributes set, but the following need to be set. + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Disable PostgreSQL on the application node + postgresql['enable'] = false + + gitlab_rails['db_host'] = 'PGBOUNCER_NODE' or 'INTERNAL_LOAD_BALANCER' + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = 'POSTGRESQL_USER_PASSWORD' + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +#### Application node post-configuration + +Ensure that all migrations ran: + +```shell +gitlab-rake gitlab:db:configure +``` + +> **Note**: If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to +PostgreSQL it may be that your PgBouncer node's IP address is missing from +PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See +[PgBouncer error `ERROR: pgbouncer cannot connect to server`](#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +in the Troubleshooting section before proceeding. + +### Ensure GitLab is running + +At this point, your GitLab instance should be up and running. Verify you are +able to login, and create issues and merge requests. If you have troubles check +the [Troubleshooting section](#troubleshooting). + +## Example configuration + +Here we'll show you some fully expanded example configurations. + +### Example recommended setup + +This example uses 3 Consul servers, 3 PgBouncer servers (with associated internal load balancer), +3 PostgreSQL servers, and 1 application node. + +We start with all servers on the same 10.6.0.0/16 private network range, they +can connect to each freely other on those addresses. + +Here is a list and description of each machine and the assigned IP: + +- `10.6.0.11`: Consul 1 +- `10.6.0.12`: Consul 2 +- `10.6.0.13`: Consul 3 +- `10.6.0.20`: Internal Load Balancer +- `10.6.0.21`: PgBouncer 1 +- `10.6.0.22`: PgBouncer 2 +- `10.6.0.23`: PgBouncer 3 +- `10.6.0.31`: PostgreSQL master +- `10.6.0.32`: PostgreSQL secondary +- `10.6.0.33`: PostgreSQL secondary +- `10.6.0.41`: GitLab application + +All passwords are set to `toomanysecrets`, please do not use this password or derived hashes and the `external_url` for GitLab is `http://gitlab.example.com`. + +Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back. + +#### Example recommended setup for Consul servers + +On each server edit `/etc/gitlab/gitlab.rb`: + +```ruby +# Disable all components except Consul +roles ['consul_role'] + +consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) +} +consul['monitoring_service_discovery'] = true +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +#### Example recommended setup for PgBouncer servers + +On each server edit `/etc/gitlab/gitlab.rb`: + +```ruby +# Disable all components except Pgbouncer and Consul agent +roles ['pgbouncer_role'] + +# Configure PgBouncer +pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + +pgbouncer['users'] = { + 'gitlab-consul': { + password: '5e0e3263571e3704ad655076301d6ebe' + }, + 'pgbouncer': { + password: '771a8625958a529132abe6f1a4acb19c' + } +} + +consul['watchers'] = %w(postgresql) +consul['enable'] = true +consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) +} +consul['monitoring_service_discovery'] = true +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +#### Internal load balancer setup + +An internal load balancer (TCP) is then required to be setup to serve each PgBouncer node (in this example on the IP of `10.6.0.20`). An example of how to do this can be found in the [PgBouncer Configure Internal Load Balancer](#configure-the-internal-load-balancer) section. + +#### Example recommended setup for PostgreSQL servers + +##### Primary node + +On primary node edit `/etc/gitlab/gitlab.rb`: + +```ruby +# Disable all components except PostgreSQL and Repmgr and Consul +roles ['postgres_role'] + +# PostgreSQL configuration +postgresql['listen_address'] = '0.0.0.0' +postgresql['hot_standby'] = 'on' +postgresql['wal_level'] = 'replica' +postgresql['shared_preload_libraries'] = 'repmgr_funcs' + +# Disable automatic database migrations +gitlab_rails['auto_migrate'] = false + +postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' +postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' +postgresql['max_wal_senders'] = 4 + +postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) +repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) + +# Configure the Consul agent +consul['services'] = %w(postgresql) +consul['enable'] = true +consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) +} +consul['monitoring_service_discovery'] = true +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +##### Secondary nodes + +On secondary nodes, edit `/etc/gitlab/gitlab.rb` and add all the configuration +added to primary node, noted above. In addition, append the following +configuration: + +```ruby +# Specify if a node should attempt to be master on initialization +repmgr['master_on_initialization'] = false +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +###### Example recommended setup for application server + +On the server edit `/etc/gitlab/gitlab.rb`: + +```ruby +external_url 'http://gitlab.example.com' + +gitlab_rails['db_host'] = '10.6.0.20' # Internal Load Balancer for PgBouncer nodes +gitlab_rails['db_port'] = 6432 +gitlab_rails['db_password'] = 'toomanysecrets' +gitlab_rails['auto_migrate'] = false + +postgresql['enable'] = false +pgbouncer['enable'] = false +consul['enable'] = true + +# Configure Consul agent +consul['watchers'] = %w(postgresql) + +pgbouncer['users'] = { + 'gitlab-consul': { + password: '5e0e3263571e3704ad655076301d6ebe' + }, + 'pgbouncer': { + password: '771a8625958a529132abe6f1a4acb19c' + } +} + +consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) +} +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +#### Example recommended setup manual steps + +After deploying the configuration follow these steps: + +1. On `10.6.0.31`, our primary database + + Enable the `pg_trgm` extension + + ```shell + gitlab-psql -d gitlabhq_production + ``` + + ```shell + CREATE EXTENSION pg_trgm; + ``` + +1. On `10.6.0.32`, our first standby database + + Make this node a standby of the primary + + ```shell + gitlab-ctl repmgr standby setup 10.6.0.21 + ``` + +1. On `10.6.0.33`, our second standby database + + Make this node a standby of the primary + + ```shell + gitlab-ctl repmgr standby setup 10.6.0.21 + ``` + +1. On `10.6.0.41`, our application server + + Set `gitlab-consul` user's PgBouncer password to `toomanysecrets` + + ```shell + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + + Run database migrations + + ```shell + gitlab-rake gitlab:db:configure + ``` + +### Example minimal setup + +This example uses 3 PostgreSQL servers, and 1 application node (with PgBouncer setup alongside). + +It differs from the [recommended setup](#example-recommended-setup) by moving the Consul servers into the same servers we use for PostgreSQL. +The trade-off is between reducing server counts, against the increased operational complexity of needing to deal with PostgreSQL [failover](#failover-procedure) and [restore](#restore-procedure) procedures in addition to [Consul outage recovery](../high_availability/consul.md#outage-recovery) on the same set of machines. + +In this example we start with all servers on the same 10.6.0.0/16 private network range, they can connect to each freely other on those addresses. + +Here is a list and description of each machine and the assigned IP: + +- `10.6.0.21`: PostgreSQL master +- `10.6.0.22`: PostgreSQL secondary +- `10.6.0.23`: PostgreSQL secondary +- `10.6.0.31`: GitLab application + +All passwords are set to `toomanysecrets`, please do not use this password or derived hashes. + +The `external_url` for GitLab is `http://gitlab.example.com` + +Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back. + +#### Example minimal configuration for database servers + +##### Primary node + +On primary database node edit `/etc/gitlab/gitlab.rb`: + +```ruby +# Disable all components except PostgreSQL, Repmgr, and Consul +roles ['postgres_role'] + +# PostgreSQL configuration +postgresql['listen_address'] = '0.0.0.0' +postgresql['hot_standby'] = 'on' +postgresql['wal_level'] = 'replica' +postgresql['shared_preload_libraries'] = 'repmgr_funcs' + +# Disable automatic database migrations +gitlab_rails['auto_migrate'] = false + +# Configure the Consul agent +consul['services'] = %w(postgresql) + +postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' +postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' +postgresql['max_wal_senders'] = 4 + +postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) +repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) + +consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23) +} +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +##### Secondary nodes + +On secondary nodes, edit `/etc/gitlab/gitlab.rb` and add all the information added +to primary node, noted above. In addition, append the following configuration + +```ruby +# Specify if a node should attempt to be master on initialization +repmgr['master_on_initialization'] = false +``` + +#### Example minimal configuration for application server + +On the server edit `/etc/gitlab/gitlab.rb`: + +```ruby +external_url 'http://gitlab.example.com' + +gitlab_rails['db_host'] = '127.0.0.1' +gitlab_rails['db_port'] = 6432 +gitlab_rails['db_password'] = 'toomanysecrets' +gitlab_rails['auto_migrate'] = false + +postgresql['enable'] = false +pgbouncer['enable'] = true +consul['enable'] = true + +# Configure PgBouncer +pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + +# Configure Consul agent +consul['watchers'] = %w(postgresql) + +pgbouncer['users'] = { + 'gitlab-consul': { + password: '5e0e3263571e3704ad655076301d6ebe' + }, + 'pgbouncer': { + password: '771a8625958a529132abe6f1a4acb19c' + } +} + +consul['configuration'] = { + retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23) +} +``` + +[Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +#### Example minimal setup manual steps + +The manual steps for this configuration are the same as for the [example recommended setup](#example-recommended-setup-manual-steps). + +### Failover procedure + +By default, if the master database fails, `repmgrd` should promote one of the +standby nodes to master automatically, and Consul will update PgBouncer with +the new master. + +If you need to failover manually, you have two options: + +**Shutdown the current master database** + +Run: + +```shell +gitlab-ctl stop postgresql +``` + +The automated failover process will see this and failover to one of the +standby nodes. + +**Or perform a manual failover** + +1. Ensure the old master node is not still active. +1. Login to the server that should become the new master and run: + + ```shell + gitlab-ctl repmgr standby promote + ``` + +1. If there are any other standby servers in the cluster, have them follow + the new master server: + + ```shell + gitlab-ctl repmgr standby follow NEW_MASTER + ``` + +### Restore procedure + +If a node fails, it can be removed from the cluster, or added back as a standby +after it has been restored to service. + +#### Remove a standby from the cluster + + From any other node in the cluster, run: + + ```shell + gitlab-ctl repmgr standby unregister --node=X + ``` + + where X is the value of node in `repmgr.conf` on the old server. + + To find this, you can use: + + ```shell + awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf + ``` + + It will output something like: + + ```plaintext + 959789412 + ``` + + Then you will use this ID to unregister the node: + + ```shell + gitlab-ctl repmgr standby unregister --node=959789412 + ``` + +#### Add a node as a standby server + + From the standby node, run: + + ```shell + gitlab-ctl repmgr standby follow NEW_MASTER + gitlab-ctl restart repmgrd + ``` + + CAUTION: **Warning:** When the server is brought back online, and before + you switch it to a standby node, repmgr will report that there are two masters. + If there are any clients that are still attempting to write to the old master, + this will cause a split, and the old master will need to be resynced from + scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. + +#### Add a failed master back into the cluster as a standby node + + Once `repmgrd` and PostgreSQL are running, the node will need to follow the new + as a standby node. + + ```shell + gitlab-ctl repmgr standby follow NEW_MASTER + ``` + + Once the node is following the new master as a standby, the node needs to be + [unregistered from the cluster on the new master node](#remove-a-standby-from-the-cluster). + + Once the old master node has been unregistered from the cluster, it will need + to be setup as a new standby: + + ```shell + gitlab-ctl repmgr standby setup NEW_MASTER + ``` + + Failure to unregister and read the old master node can lead to subsequent failovers + not working. + +### Alternate configurations + +#### Database authorization + +By default, we give any host on the database network the permission to perform +repmgr operations using PostgreSQL's `trust` method. If you do not want this +level of trust, there are alternatives. + +You can trust only the specific nodes that will be database clusters, or you +can require md5 authentication. + +#### Trust specific addresses + +If you know the IP address, or FQDN of all database and PgBouncer nodes in the +cluster, you can trust only those nodes. + +In `/etc/gitlab/gitlab.rb` on all of the database nodes, set +`repmgr['trust_auth_cidr_addresses']` to an array of strings containing all of +the addresses. + +If setting to a node's FQDN, they must have a corresponding PTR record in DNS. +If setting to a node's IP address, specify it as `XXX.XXX.XXX.XXX/32`. + +For example: + +```ruby +repmgr['trust_auth_cidr_addresses'] = %w(192.168.1.44/32 db2.example.com) +``` + +#### MD5 Authentication + +If you are running on an untrusted network, repmgr can use md5 authentication +with a [`.pgpass` file](https://www.postgresql.org/docs/11/libpq-pgpass.html) +to authenticate. + +You can specify by IP address, FQDN, or by subnet, using the same format as in +the previous section: + +1. On the current master node, create a password for the `gitlab` and + `gitlab_repmgr` user: + + ```shell + gitlab-psql -d template1 + template1=# \password gitlab_repmgr + Enter password: **** + Confirm password: **** + template1=# \password gitlab + ``` + +1. On each database node: + + 1. Edit `/etc/gitlab/gitlab.rb`: + 1. Ensure `repmgr['trust_auth_cidr_addresses']` is **not** set + 1. Set `postgresql['md5_auth_cidr_addresses']` to the desired value + 1. Set `postgresql['sql_replication_user'] = 'gitlab_repmgr'` + 1. Reconfigure with `gitlab-ctl reconfigure` + 1. Restart PostgreSQL with `gitlab-ctl restart postgresql` + + 1. Create a `.pgpass` file. Enter the `gitlab_repmgr` password twice to + when asked: + + ```shell + gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' + ``` + +1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`: + 1. Ensure `gitlab_rails['db_password']` is set to the plaintext password for + the `gitlab` database user + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect + +## Troubleshooting + +### Consul and PostgreSQL changes not taking effect + +Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it will not restart the services. However, not all changes can be activated by reloading. + +To restart either service, run `gitlab-ctl restart SERVICE` + +For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. + +On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../high_availability/consul.md#restarting-the-server-cluster) for instructions on how to restart the service. + +### `gitlab-ctl repmgr-check-master` command produces errors + +If this command displays errors about database permissions it is likely that something failed during +install, resulting in the `gitlab-consul` database user getting incorrect permissions. Follow these +steps to fix the problem: + +1. On the master database node, connect to the database prompt - `gitlab-psql -d template1` +1. Delete the `gitlab-consul` user - `DROP USER "gitlab-consul";` +1. Exit the database prompt - `\q` +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) and the user will be re-added with the proper permissions. +1. Change to the `gitlab-consul` user - `su - gitlab-consul` +1. Try the check command again - `gitlab-ctl repmgr-check-master`. + +Now there should not be errors. If errors still occur then there is another problem. + +### PgBouncer error `ERROR: pgbouncer cannot connect to server` + +You may get this error when running `gitlab-rake gitlab:db:configure` or you +may see the error in the PgBouncer log file. + +```plaintext +PG::ConnectionBad: ERROR: pgbouncer cannot connect to server +``` + +The problem may be that your PgBouncer node's IP address is not included in the +`trust_auth_cidr_addresses` setting in `/etc/gitlab/gitlab.rb` on the database nodes. + +You can confirm that this is the issue by checking the PostgreSQL log on the master +database node. If you see the following error then `trust_auth_cidr_addresses` +is the problem. + +```plaintext +2018-03-29_13:59:12.11776 FATAL: no pg_hba.conf entry for host "123.123.123.123", user "pgbouncer", database "gitlabhq_production", SSL off +``` + +To fix the problem, add the IP address to `/etc/gitlab/gitlab.rb`. + +```ruby +postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>) +``` + +[Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +### Issues with other components + +If you're running into an issue with a component not outlined here, be sure to check the troubleshooting section of their specific documentation page. + +- [Consul](../high_availability/consul.md#troubleshooting) +- [PostgreSQL](https://docs.gitlab.com/omnibus/settings/database.html#troubleshooting) +- [GitLab application](../high_availability/gitlab.md#troubleshooting) diff --git a/doc/administration/postgresql/standalone.md b/doc/administration/postgresql/standalone.md new file mode 100644 index 00000000000..3e7826ce009 --- /dev/null +++ b/doc/administration/postgresql/standalone.md @@ -0,0 +1,65 @@ +# Standalone PostgreSQL using Omnibus GitLab **(CORE ONLY)** + +If you wish to have your database service hosted separately from your GitLab +application server(s), you can do this using the PostgreSQL binaries packaged +together with Omnibus GitLab. This is recommended as part of our +[reference architecture for up to 2,000 users](../reference_architectures/2k_users.md). + +## Setting it up + +1. SSH into the PostgreSQL server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Do not complete any other steps on the download page. +1. Generate a password hash for PostgreSQL. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `POSTGRESQL_PASSWORD_HASH`. + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents below, updating placeholder + values appropriately. + + - `POSTGRESQL_PASSWORD_HASH` - The value output from the previous step + - `APPLICATION_SERVER_IP_BLOCKS` - A space delimited list of IP subnets or IP + addresses of the GitLab application servers that will connect to the + database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` + + ```ruby + # Disable all components except PostgreSQL + roles ['postgres_role'] + repmgr['enable'] = false + consul['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + redis_exporter['enable'] = false + gitlab_exporter['enable'] = false + + postgresql['listen_address'] = '0.0.0.0' + postgresql['port'] = 5432 + + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + # ???? + postgresql['trust_auth_cidr_addresses'] = %w(APPLICATION_SERVER_IP_BLOCKS) + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + ``` + + NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Note the PostgreSQL node's IP address or hostname, port, and + plain text password. These will be necessary when configuring the GitLab + application servers later. +1. [Enable monitoring](replication_and_failover.md#enable-monitoring) + +Advanced configuration options are supported and can be added if +needed. diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md index 83a3d2c8884..a46a2b34687 100644 --- a/doc/administration/raketasks/github_import.md +++ b/doc/administration/raketasks/github_import.md @@ -12,6 +12,11 @@ Bear in mind that the syntax is very specific. Remove any spaces within the argu before/after the brackets. Also, some shells (for example, `zsh`) can interpret the open/close brackets (`[]`) separately. You may need to either escape the brackets or use double quotes. +## Caveats + +If the GitHub [rate limit](https://developer.github.com/v3/#rate-limiting) is reached while importing, +the importing process will wait (`sleep()`) until it can continue importing. + ## Importing multiple projects To import a project from the list of your GitHub projects available: diff --git a/doc/administration/raketasks/ldap.md b/doc/administration/raketasks/ldap.md index 2871a9235a3..d168e3d568c 100644 --- a/doc/administration/raketasks/ldap.md +++ b/doc/administration/raketasks/ldap.md @@ -32,13 +32,13 @@ rake gitlab:ldap:check[50] > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/14735) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.2. -The following task will run a [group sync](../auth/ldap-ee.md#group-sync) immediately. This is valuable +The following task will run a [group sync](../auth/ldap/index.md#group-sync-starter-only) immediately. This is valuable when you'd like to update all configured group memberships against LDAP without waiting for the next scheduled group sync to be run. NOTE: **NOTE:** If you'd like to change the frequency at which a group sync is performed, -[adjust the cron schedule](../auth/ldap-ee.md#adjusting-ldap-group-sync-schedule) +[adjust the cron schedule](../auth/ldap/index.md#adjusting-ldap-group-sync-schedule-starter-only) instead. **Omnibus Installation** diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index eee68c0da0a..78094f00a43 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -51,6 +51,40 @@ Hooks: /home/git/gitlab-shell/hooks/ Git: /usr/bin/git ``` +## Show GitLab license information **(STARTER ONLY)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20501) in GitLab Starter 12.6. + +This command shows information about your [GitLab license](../../user/admin_area/license.md) and +how many seats are used. It is only available on GitLab Enterprise +installations: a license cannot be installed into GitLab Community Edition. + +These may be useful when raising tickets with Support, or for programmatically +checking your license parameters. + +**Omnibus Installation** + +```shell +sudo gitlab-rake gitlab:license:info +``` + +**Source Installation** + +```shell +bundle exec rake gitlab:license:info RAILS_ENV=production +``` + +Example output: + +```plaintext +Today's Date: 2020-02-29 +Current User Count: 30 +Max Historical Count: 30 +Max Users in License: 40 +License valid from: 2019-11-29 to 2020-11-28 +Email associated with license: user@example.com +``` + ## Check GitLab configuration The `gitlab:check` Rake task runs the following Rake tasks: @@ -62,7 +96,7 @@ The `gitlab:check` Rake task runs the following Rake tasks: It will check that each component was set up according to the installation guide and suggest fixes for issues found. This command must be run from your application server and will not work correctly on -component servers like [Gitaly](../gitaly/index.md#running-gitaly-on-its-own-server). +component servers like [Gitaly](../gitaly/index.md#run-gitaly-on-its-own-server). You may also have a look at our troubleshooting guides for: @@ -265,6 +299,20 @@ database: gitlabhq_production up migration_id migration_name ``` +## Run incomplete database migrations + +Database migrations can be stuck in an incomplete state. That is, they'll have a `down` +status in the output of the `sudo gitlab-rake db:migrate:status` command. + +To complete these migrations, use the following Rake task: + +```shell +sudo gitlab-rake db:migrate +``` + +After the command completes, run `sudo gitlab-rake db:migrate:status` to check if all +migrations are completed (have an `up` status). + ## Import common metrics Sometimes you may need to re-import the common metrics that power the Metrics dashboards. diff --git a/doc/administration/raketasks/praefect.md b/doc/administration/raketasks/praefect.md index c3dadb6bc30..c48e23df77a 100644 --- a/doc/administration/raketasks/praefect.md +++ b/doc/administration/raketasks/praefect.md @@ -7,7 +7,7 @@ Prints out checksums of the repository of a given project_id on the primary as well as secondary internal Gitaly nodes. NOTE: **Note:** -This only is relevant and works for projects that have been created on a praefect storage. See the [Praefect Documentation](../gitaly/praefect.md) for configuring Praefect. +This only is relevant and works for projects that have been created on a Praefect storage. See the [Praefect Documentation](../gitaly/praefect.md) for configuring Praefect. **Omnibus Installation** diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index 2ab8b13e29e..2e65e889c90 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -1,6 +1,6 @@ # Project import/export administration **(CORE ONLY)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/3050) in GitLab 8.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3050) in GitLab 8.9. > - From GitLab 11.3, import/export can use object storage automatically. GitLab provides Rake tasks relating to project import and export. For more information, see: diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md index 30f50c24138..74fd2c2ebb6 100644 --- a/doc/administration/raketasks/storage.md +++ b/doc/administration/raketasks/storage.md @@ -104,7 +104,7 @@ You can monitor the progress in the **{admin}** **Admin Area > Monitoring > Back There is a specific queue you can watch to see how long it will take to finish: `hashed_storage:hashed_storage_project_migrate`. -After it reaches zero, you can confirm every project has been migrated by running the commands bellow. +After it reaches zero, you can confirm every project has been migrated by running the commands below. If you find it necessary, you can run this migration script again to schedule missing projects. Any error or warning will be logged in Sidekiq's log file. diff --git a/doc/administration/raketasks/uploads/migrate.md b/doc/administration/raketasks/uploads/migrate.md index d58b802b024..b164c4e744d 100644 --- a/doc/administration/raketasks/uploads/migrate.md +++ b/doc/administration/raketasks/uploads/migrate.md @@ -128,7 +128,7 @@ CAUTION: **Warning:** **Extended downtime is required** so no new files are created in object storage during the migration. A configuration setting will be added soon to allow migrating from object storage to local files with only a brief moment of downtime for configuration changes. -To follow progress, see the [relevant issue](https://gitlab.com/gitlab-org/gitlab/issues/30979). +To follow progress, see the [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/30979). ### All-in-one Rake task diff --git a/doc/administration/raketasks/uploads/sanitize.md b/doc/administration/raketasks/uploads/sanitize.md index 7c760b95c33..9586ab2c6f4 100644 --- a/doc/administration/raketasks/uploads/sanitize.md +++ b/doc/administration/raketasks/uploads/sanitize.md @@ -65,5 +65,5 @@ The output is written into an `exif.log` file because it will probably be long. If sanitization fails for an upload, an error message should be in the output of the Rake task. Typical reasons include that the file is missing in the storage or it's not a valid image. -[Report](https://gitlab.com/gitlab-org/gitlab/issues/new) any issues and use the prefix 'EXIF' in +[Report](https://gitlab.com/gitlab-org/gitlab/-/issues/new) any issues and use the prefix 'EXIF' in the issue title with the error output and (if possible) the image. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 7f31f336251..a15fcf722a5 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -8,8 +8,8 @@ For a full list of reference architectures, see > - **High Availability:** True > - **Test RPS rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|------------------------| +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | +|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|----------------| | GitLab Rails ([1](#footnotes)) | 3 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | | PostgreSQL | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | | PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | @@ -56,7 +56,7 @@ For a full list of reference architectures, see 1. NFS can be used as an alternative for both repository data (replacing Gitaly) and object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). 1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets @@ -74,6 +74,3 @@ For a full list of reference architectures, see or higher, are required for your CPU or Node counts accordingly. For more information, a [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index b6aaffa9488..34805a8ac68 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -7,11 +7,18 @@ For a full list of reference architectures, see > - **Supported users (approximate):** 1,000 > - **High Availability:** False -| Users | Configuration([8](#footnotes)) | GCP | AWS([9](#footnotes)) | Azure([9](#footnotes)) | -|-------|--------------------------------|---------------|----------------------|------------------------| -| 100 | 2 vCPU, 7.2GB Memory | n1-standard-2 | m5.large | D2s v3 | -| 500 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| 1000 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | D8s v3 | +| Users | Configuration([8](#footnotes)) | GCP | AWS | Azure | +|-------|------------------------------------|----------------|---------------------|------------------------| +| 500 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | +| 1000 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | + +In addition to the above, we recommend having at least +2GB of swap on your server, even if you currently have +enough available RAM. Having swap will help reduce the chance of errors occurring +if your available memory changes. We also recommend +configuring the kernel's swappiness setting +to a low value like `10` to make the most of your RAM while still having the swap +available when needed. For situations where you need to serve up to 1,000 users, a single-node solution with [frequent backups](index.md#automated-backups-core-only) is appropriate @@ -25,7 +32,7 @@ requirements, this is the ideal solution. NOTE: **Note:** You can also optionally configure GitLab to use an -[external PostgreSQL service](../external_database.md) or an +[external PostgreSQL service](../postgresql/external.md) or an [external object storage service](../high_availability/object_storage.md) for added performance and reliability at a reduced complexity cost. @@ -59,7 +66,7 @@ added performance and reliability at a reduced complexity cost. 1. NFS can be used as an alternative for both repository data (replacing Gitaly) and object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). 1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets @@ -77,6 +84,3 @@ added performance and reliability at a reduced complexity cost. or higher, are required for your CPU or Node counts accordingly. For more information, a [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 2ee692d635c..d851fa124c6 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -8,23 +8,23 @@ For a full list of reference architectures, see > - **High Availability:** True > - **Test RPS rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|------------------------| -| GitLab Rails ([1](#footnotes)) | 5 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | -| PostgreSQL | 3 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | D8s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 32 vCPU, 120GB Memory | n1-standard-32 | m5.8xlarge | D32s v3 | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | +|--------------------------------------------------------------|-------|---------------------------------|------------------|-----------------------|----------------| +| GitLab Rails ([1](#footnotes)) | 5 | 32 vCPU, 28.8GB Memory | `n1-highcpu-32` | `c5.9xlarge` | F32s v2 | +| PostgreSQL | 3 | 8 vCPU, 30GB Memory | `n1-standard-8` | `m5.2xlarge` | D8s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 32 vCPU, 120GB Memory | `n1-standard-32` | `m5.8xlarge` | D32s v3 | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | +| Consul | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | | Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | ## Footnotes @@ -56,7 +56,7 @@ For a full list of reference architectures, see 1. NFS can be used as an alternative for both repository data (replacing Gitaly) and object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). 1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets @@ -74,6 +74,3 @@ For a full list of reference architectures, see or higher, are required for your CPU or Node counts accordingly. For more information, a [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 874e00e6722..0a3ade1acf1 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -1,3 +1,7 @@ +--- +reading_time: true +--- + # Reference architecture: up to 2,000 users This page describes GitLab reference architecture for up to 2,000 users. @@ -6,85 +10,864 @@ For a full list of reference architectures, see > - **Supported users (approximate):** 2,000 > - **High Availability:** False -> - **Test RPS rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|----------------| -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| PostgreSQL | 1 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | -| Redis ([3](#footnotes)) | 1 | 1 vCPU, 3.75GB Memory | n1-standard-1 | m5.large | D2s v3 | -| Gitaly ([5](#footnotes)) ([7](#footnotes)) | X ([2](#footnotes)) | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| GitLab Rails ([1](#footnotes)) | 2 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | - -## Setup instructions - -1. [Configure the external load balancing node](../high_availability/load_balancer.md) - that will handle the load balancing of the two GitLab application services nodes. -1. [Configure the Object Storage](../object_storage.md) ([4](#footnotes)) used for shared data objects. -1. (Optional) [Configure NFS](../high_availability/nfs.md) to have - shared disk storage service as an alternative to Gitaly and/or - [Object Storage](../object_storage.md) (although not recommended). - NFS is required for GitLab Pages, you can skip this step if you're not using that feature. -1. [Configure PostgreSQL](../high_availability/load_balancer.md), the database for GitLab. -1. [Configure Redis](../high_availability/redis.md). -1. [Configure Gitaly](../gitaly/index.md#running-gitaly-on-its-own-server), - which is used to provide access to the Git repositories. -1. [Configure the main GitLab Rails application](../high_availability/gitlab.md) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all - frontend requests (UI, API, Git over HTTP/SSH). -1. [Configure Prometheus](../high_availability/monitoring_node.md) to monitor your GitLab environment. - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with four threads. For - nodes that are running Rails with other components the worker value should be reduced - accordingly where we've found 50% achieves a good balance but this is dependent - on workload. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum for HA environments - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (less than 3,000 users) a single instance should suffice. - For medium sized installs (3,000 - 5,000) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend an [Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. Although other load balancers with similar feature sets - could also be used, those load balancers have not been validated. - -1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. +> - **Test requests per second (RPS) rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS + +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------------------------|-----------|---------------------------------|---------------|-----------------------|----------------| +| Load balancer | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | +| Object storage | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| PostgreSQL | 1 | 2 vCPU, 7.5GB memory | n1-standard-2 | m5.large | D2s v3 | +| Redis | 1 | 1 vCPU, 3.75GB memory | n1-standard-1 | m5.large | D2s v3 | +| Gitaly | 1 | 4 vCPU, 15GB memory | n1-standard-4 | m5.xlarge | D4s v3 | +| GitLab Rails | 2 | 8 vCPU, 7.2GB memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB memory | n1-highcpu-2 | c5.large | F2s v2 | + +The Google Cloud Platform (GCP) architectures were built and tested using the +[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) +CPU platform. On different hardware you may find that adjustments, either lower +or higher, are required for your CPU or node counts. For more information, see +our [Sysbench](https://github.com/akopytov/sysbench)-based +[CPU benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +AWS-equivalent and Azure-equivalent configurations are rough suggestions that +may change in the future, and haven't been tested or validated. + +Due to better performance and availability, for data objects (such as LFS, +uploads, or artifacts), using an [object storage service](#configure-the-object-storage) +is recommended instead of using NFS. Using an object storage service also +doesn't require you to provision and maintain a node. + +## Setup components + +To set up GitLab and its components to accommodate up to 2,000 users: + +1. [Configure the external load balancing node](#configure-the-load-balancer) + to handle the load balancing of the two GitLab application services nodes. +1. [Configure the object storage](#configure-the-object-storage) used for + shared data objects. +1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) + to have shared disk storage service as an alternative to Gitaly or object + storage. You can skip this step if you're not using GitLab Pages (which + requires NFS). +1. [Configure PostgreSQL](#configure-postgresql), the database for GitLab. +1. [Configure Redis](#configure-redis). +1. [Configure Gitaly](#configure-gitaly), which provides access to the Git + repositories. +1. [Configure the main GitLab Rails application](#configure-gitlab-rails) + to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + requests (which include UI, API, and Git over HTTP/SSH). +1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab + environment. + +## Configure the load balancer + +NOTE: **Note:** +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/). +Although you can use a load balancer with a similar set of features, GitLab +hasn't validated other load balancers. + +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics for which load balancer to +use or its exact configuration is out of scope for the GitLab documentation. +If you're managing multi-node systems (including GitLab) you'll probably +already have a load balancer of choice. Some examples including HAProxy +(open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation +includes the ports and protocols for use with GitLab. + +The next question is how you will handle SSL in your environment. There are +several different options: + +- [The application node terminates SSL](#application-node-terminates-ssl). +- [The load balancer terminates SSL without backend SSL](#load-balancer-terminates-ssl-without-backend-ssl) + and communication is not secure between the load balancer and the application node. +- [The load balancer terminates SSL with backend SSL](#load-balancer-terminates-ssl-with-backend-ssl) + and communication is *secure* between the load balancer and the application node. + +### Application node terminates SSL + +Configure your load balancer to pass connections on port 443 as `TCP` instead +of `HTTP(S)`. This will pass the connection unaltered to the application node's +NGINX service, which has the SSL certificate and listens to port 443. + +For details about managing SSL certificates and configuring NGINX, see the +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). + +### Load balancer terminates SSL without backend SSL + +Configure your load balancer to use the `HTTP(S)` protocol instead of `TCP`. +The load balancer will be responsible for both managing SSL certificates and +terminating SSL. + +Due to communication between the load balancer and GitLab not being secure, +you'll need to complete some additional configuration. For details, see the +[NGINX proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl). + +### Load balancer terminates SSL with backend SSL + +Configure your load balancers (or single balancer, if you have only one) to use +the `HTTP(S)` protocol rather than `TCP`. The load balancers will be +responsible for the managing SSL certificates for end users. + +Traffic will be secure between the load balancers and NGINX in this scenario, +and there's no need to add a configuration for proxied SSL. However, you'll +need to add a configuration to GitLab to configure SSL certificates. For +details about managing SSL certificates and configuring NGINX, see the +[NGINX HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). + +### Ports + +The basic load balancer ports you should use are described in the following +table: + +| Port | Backend Port | Protocol | +| ------- | ------------ | ------------------------ | +| 80 | 80 | HTTP (*1*) | +| 443 | 443 | TCP or HTTPS (*1*) (*2*) | +| 22 | 22 | TCP | + +- (*1*): [Web terminal](../../ci/environments/index.md#web-terminals) support + requires your load balancer to correctly handle WebSocket connections. + When using HTTP or HTTPS proxying, your load balancer must be configured + to pass through the `Connection` and `Upgrade` hop-by-hop headers. For + details, see the [web terminal](../integration/terminal.md) integration guide. +- (*2*): When using the HTTPS protocol for port 443, you'll need to add an SSL + certificate to the load balancers. If you need to terminate SSL at the + GitLab application server, use the TCP protocol. + +If you're using GitLab Pages with custom domain support you will need some +additional port configurations. GitLab Pages requires a separate virtual IP +address. Configure DNS to point the `pages_external_url` from +`/etc/gitlab/gitlab.rb` to the new virtual IP address. For more information, +see the [GitLab Pages documentation](../pages/index.md). + +| Port | Backend Port | Protocol | +| ------- | ------------- | --------- | +| 80 | Varies (*1*) | HTTP | +| 443 | Varies (*1*) | TCP (*2*) | + +- (*1*): The backend port for GitLab Pages depends on the + `gitlab_pages['external_http']` and `gitlab_pages['external_https']` + settings. For details, see the [GitLab Pages documentation](../pages/index.md). +- (*2*): Port 443 for GitLab Pages must use the TCP protocol. Users can + configure custom domains with custom SSL, which wouldn't be possible if SSL + was terminated at the load balancer. + +#### Alternate SSH Port + +Some organizations have policies against opening SSH port 22. In this case, +it may be helpful to configure an alternate SSH hostname that instead allows +users to use SSH over port 443. An alternate SSH hostname requires a new +virtual IP address compared to the previously described GitLab HTTP +configuration. + +Configure DNS for an alternate SSH hostname, such as `altssh.gitlab.example.com`: + +| LB Port | Backend Port | Protocol | +| ------- | ------------ | -------- | +| 443 | 22 | TCP | + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure the object storage + +GitLab supports using an object storage service for holding several types of +data, and is recommended over [NFS](#configure-nfs-optional). In general, +object storage services are better for larger environments, as object storage +is typically much more performant, reliable, and scalable. + +Object storage options that GitLab has either tested or is aware of customers +using, includes: + +- SaaS/Cloud solutions (such as [Amazon S3](https://aws.amazon.com/s3/) or + [Google Cloud Storage](https://cloud.google.com/storage)). +- On-premises hardware and appliances, from various storage vendors. +- MinIO ([Deployment guide](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html)). + +To configure GitLab to use object storage, refer to the following guides based +on the features you intend to use: + +1. [Object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). +1. [Object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [incremental logging](../job_logs.md#new-incremental-logging-architecture). +1. [Object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). +1. [Object storage for uploads](../uploads.md#using-object-storage-core-only). +1. [Object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). +1. [Object storage for Container Registry](../packages/container_registry.md#container-registry-storage-driver) (optional feature). +1. [Object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). +1. [Object storage for packages](../packages/index.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. [Object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** +1. [Object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** +1. [Object storage for autoscale Runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional, for improved performance). +1. [Object storage for Terraform state files](../terraform_state.md#using-object-storage-core-only). + +Using separate buckets for each data type is the recommended approach for GitLab. + +A limitation of our configuration is that each use of object storage is +separately configured. We have an [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/23345) +for improving this, which would allow for one bucket with separate folders. + +Using a single bucket when GitLab is deployed with the Helm chart causes +restoring from a backup to +[not function properly](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer). +Although you may not be using a Helm deployment right now, if you migrate +GitLab to a Helm deployment later, GitLab would still work, but you may not +realize backups aren't working correctly until a critical requirement for +functioning backups is encountered. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure NFS (optional) + +For improved performance, [object storage](#configure-the-object-storage), +along with [Gitaly](#configure-gitaly), are recommended over using NFS whenever +possible. However, if you intend to use GitLab Pages, +[you must use NFS](troubleshooting.md#gitlab-pages-requires-nfs). + +For information about configuring NFS, see the [NFS documentation page](../high_availability/nfs.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure PostgreSQL + +In this section, you'll be guided through configuring an external PostgreSQL database +to be used with GitLab. + +### Provide your own PostgreSQL instance + +If you're hosting GitLab on a cloud provider, you can optionally use a +managed service for PostgreSQL. For example, AWS offers a managed relational +database service (RDS) that runs PostgreSQL. + +If you use a cloud-managed service, or provide your own PostgreSQL: + +1. Set up PostgreSQL according to the + [database requirements document](../../install/requirements.md#database). +1. Create a `gitlab` username with a password of your choice. The `gitlab` user + needs privileges to create the `gitlabhq_production` database. +1. Configure the GitLab application servers with the appropriate details. + This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). + +### Standalone PostgreSQL using Omnibus GitLab + +1. SSH into the PostgreSQL server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Do not complete any other steps on the download page. +1. Generate a password hash for PostgreSQL. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `POSTGRESQL_PASSWORD_HASH`. + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents below, updating placeholder + values appropriately. + + - `POSTGRESQL_PASSWORD_HASH` - The value output from the previous step + - `APPLICATION_SERVER_IP_BLOCKS` - A space delimited list of IP subnets or IP + addresses of the GitLab application servers that will connect to the + database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` + + ```ruby + # Disable all components except PostgreSQL + roles ['postgres_role'] + repmgr['enable'] = false + consul['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + redis_exporter['enable'] = false + gitlab_exporter['enable'] = false + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + postgres_exporter['dbname'] = 'gitlabhq_production' + postgres_exporter['password'] = 'POSTGRESQL_PASSWORD_HASH' + + # Set the PostgreSQL address and port + postgresql['listen_address'] = '0.0.0.0' + postgresql['port'] = 5432 + + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + + # Replace APPLICATION_SERVER_IP_BLOCK with the CIDR address of the application node + postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 APPLICATION_SERVER_IP_BLOCK) + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Note the PostgreSQL node's IP address or hostname, port, and + plain text password. These will be necessary when configuring the [GitLab + application server](#configure-gitlab-rails) later. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/database.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Redis + +In this section, you'll be guided through configuring an external Redis instance +to be used with GitLab. + +### Provide your own Redis instance + +Redis version 5.0 or higher is required, as this is what ships with +Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions +do not support an optional count argument to SPOP which is now required for +[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). + +In addition, GitLab makes use of certain commands like `UNLINK` and `USAGE` which +were introduced only in Redis 4. + +Managed Redis from cloud providers such as AWS ElastiCache will work. If these +services support high availability, be sure it is not the Redis Cluster type. + +Note the Redis node's IP address or hostname, port, and password (if required). +These will be necessary when configuring the +[GitLab application servers](#configure-gitlab-rails) later. + +### Standalone Redis using Omnibus GitLab + +The Omnibus GitLab package can be used to configure a standalone Redis server. +The steps below are the minimum necessary to configure a Redis server with +Omnibus: + +1. SSH into the Redis server. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + ## Enable Redis + redis['enable'] = true + + ## Disable all other services + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + postgresql['enable'] = false + nginx['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + gitlab_exporter['enable'] = false + gitaly['enable'] = false + grafana['enable'] = false + + redis['bind'] = '0.0.0.0' + redis['port'] = 6379 + redis['password'] = 'SECRET_PASSWORD_HERE' + + gitlab_rails['enable'] = false + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://0.0.0.0:6379', + 'redis.password' => 'SECRET_PASSWORD_HERE', + } + ``` + +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Note the Redis node's IP address or hostname, port, and + Redis password. These will be necessary when [configuring the GitLab + application servers](#configure-gitlab-rails) later. + +Advanced [configuration options](https://docs.gitlab.com/omnibus/settings/redis.html) +are supported and can be added if needed. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Gitaly + +Deploying Gitaly in its own server can benefit GitLab installations that are +larger than a single machine. Gitaly node requirements are dependent on data, +specifically the number of projects and their sizes. It's recommended that each +Gitaly node store no more than 5TB of data. Your 2K setup may require one or more +nodes depending on your repository storage requirements. + +We strongly recommend that all Gitaly nodes should be set up with SSD disks with a throughput of at least +8,000 IOPS for read operations and 2,000 IOPS for write, as Gitaly has heavy I/O. +These IOPS values are recommended only as a starter as with time they may be +adjusted higher or lower depending on the scale of your environment's workload. +If you're running the environment on a Cloud provider +you may need to refer to their documentation on how configure IOPS correctly. + +Some things to note: + +- The GitLab Rails application shards repositories into [repository storages](../repository_storage_paths.md). +- A Gitaly server can host one or more storages. +- A GitLab server can use one or more Gitaly servers. +- Gitaly addresses must be specified in such a way that they resolve + correctly for ALL Gitaly clients. +- Gitaly servers must not be exposed to the public internet, as Gitaly's network + traffic is unencrypted by default. The use of a firewall is highly recommended + to restrict access to the Gitaly server. Another option is to + [use TLS](#gitaly-tls-support). + +TIP: **Tip:** +For more information about Gitaly's history and network architecture see the +[standalone Gitaly documentation](../gitaly/index.md). + +Note: **Note:** The token referred to throughout the Gitaly documentation is +just an arbitrary password selected by the administrator. It is unrelated to +tokens created for the GitLab API or other similar web API tokens. + +Below we describe how to configure one Gitaly server `gitaly1.internal` with +secret token `gitalysecret`. We assume your GitLab installation has two +repository storages: `default` and `storage1`. + +To configure the Gitaly server: + +1. [Download/Install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page but + **without** providing the `EXTERNAL_URL` value. +1. Edit `/etc/gitlab/gitlab.rb` to configure storage paths, enable + the network listener and configure the token: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + # /etc/gitlab/gitlab.rb + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the GitLab Rails application setup + gitaly['auth_token'] = 'gitlaysecret' + gitlab_shell['secret_token'] = 'shellsecret' + + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + grafana['enable'] = false + + # If you run a seperate monitoring node you can disable these services + alertmanager['enable'] = false + prometheus['enable'] = false + + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + gitaly['listen_addr'] = "0.0.0.0:8075" + gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + ``` + +1. Append the following to `/etc/gitlab/gitlab.rb` on `gitaly1.internal`: + + ```ruby + git_data_dirs({ + 'default' => { + 'path' => '/var/opt/gitlab/git-data' + }, + 'storage1' => { + 'path' => '/mnt/gitlab/git-data' + }, + }) + ``` + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Confirm that Gitaly can perform callbacks to the internal API: + + ```shell + sudo /opt/gitlab/embedded/service/gitlab-shell/bin/check -config /opt/gitlab/embedded/service/gitlab-shell/config.yml + ``` + +### Gitaly TLS support + +Gitaly supports TLS encryption. To be able to communicate +with a Gitaly instance that listens for secure connections you will need to use `tls://` URL +scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. + +You will need to bring your own certificates as this isn't provided automatically. +The certificate, or its certificate authority, must be installed on all Gitaly +nodes (including the Gitaly node using the certificate) and on all client nodes +that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). + +NOTE: **Note** +The self-signed certificate must specify the address you use to access the +Gitaly server. If you are addressing the Gitaly server by a hostname, you can +either use the Common Name field for this, or add it as a Subject Alternative +Name. If you are addressing the Gitaly server by its IP address, you must add it +as a Subject Alternative Name to the certificate. +[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). + +NOTE: **Note:** +It is possible to configure Gitaly servers with both an +unencrypted listening address `listen_addr` and an encrypted listening +address `tls_listen_addr` at the same time. This allows you to do a +gradual transition from unencrypted to encrypted traffic, if necessary. + +To configure Gitaly with TLS: + +1. Create the `/etc/gitlab/ssl` directory and copy your key and certificate there: + + ```shell + sudo mkdir -p /etc/gitlab/ssl + sudo chmod 755 /etc/gitlab/ssl + sudo cp key.pem cert.pem /etc/gitlab/ssl/ + sudo chmod 644 key.pem cert.pem + ``` + +1. Copy the cert to `/etc/gitlab/trusted-certs` so Gitaly will trust the cert when + calling into itself: + + ```shell + sudo cp /etc/gitlab/ssl/cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + <!-- + updates to following example must also be made at + https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> + + ```ruby + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" + gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + ``` + +1. Delete `gitaly['listen_addr']` to allow only encrypted connections. +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure GitLab Rails + +NOTE: **Note:** +In our architectures we run each GitLab Rails node using the Puma webserver +and have its number of workers set to 90% of available CPUs along with four threads. For +nodes that are running Rails with other components the worker value should be reduced +accordingly where we've found 50% achieves a good balance but this is dependent +on workload. + +This section describes how to configure the GitLab application (Rails) component. +On each node perform the following: + +1. If you're [using NFS](#configure-nfs-optional): + + 1. If necessary, install the NFS client utility packages using the following + commands: + + ```shell + # Ubuntu/Debian + apt-get install nfs-common + + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` + + 1. Specify the necessary NFS mounts in `/etc/fstab`. + The exact contents of `/etc/fstab` will depend on how you chose + to configure your NFS server. See the [NFS documentation](../high_availability/nfs.md) + for examples and the various options. + + 1. Create the shared directories. These may be different depending on your NFS + mount locations. + + ```shell + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` + +1. Download/install Omnibus GitLab using **steps 1 and 2** from + [GitLab downloads](https://about.gitlab.com/install/). Do not complete other + steps on the download page. +1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. + To maintain uniformity of links across nodes, the `external_url` + on the application server should point to the external URL that users will use + to access GitLab. This would be the URL of the [load balancer](#configure-the-load-balancer) + which will route traffic to the GitLab application server: + + ```ruby + external_url 'https://gitlab.example.com' + + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the Gitaly setup + gitlab_rails['gitaly_token'] = 'gitalyecret' + gitlab_shell['secret_token'] = 'shellsecret' + + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, + 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, + }) + + ## Disable components that will not be on the GitLab application server + roles ['application_role'] + gitaly['enable'] = false + nginx['enable'] = true + + ## PostgreSQL connection details + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server + gitlab_rails['db_password'] = 'DB password' + + ## Redis connection details + gitlab_rails['redis_port'] = '6379' + gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server + gitlab_rails['redis_password'] = 'Redis Password' + + # Set the network addresses that the exporters used for monitoring will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' + sidekiq['listen_address'] = "0.0.0.0" + puma['listen'] = '0.0.0.0' + + # Add the monitoring node's IP address to the monitoring whitelist and allow it to + # scrape the NGINX metrics. Replace placeholder `monitoring.gitlab.example.com` with + # the address and/or subnets gathered from the monitoring node + gitlab_rails['monitoring_whitelist'] = ['<MONITOR NODE IP>/32', '127.0.0.0/8'] + nginx['status']['options']['allow'] = ['<MONITOR NODE IP>/32', '127.0.0.0/8'] + + ## Uncomment and edit the following options if you have set up NFS + ## + ## Prevent GitLab from starting if NFS data mounts are not available + ## + #high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + ## + ## Ensure UIDs and GIDs match between servers for permissions via NFS + ## + #user['uid'] = 9000 + #user['gid'] = 9000 + #web_server['uid'] = 9001 + #web_server['gid'] = 9001 + #registry['uid'] = 9002 + #registry['gid'] = 9002 + ``` + +1. If you're using [Gitaly with TLS support](#gitaly-tls-support), make sure the + `git_data_dirs` entry is configured with `tls` instead of `tcp`: + + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage1' => { 'gitaly_address' => 'tls://gitaly1.internal:9999' }, + 'storage2' => { 'gitaly_address' => 'tls://gitaly2.internal:9999' }, + }) + ``` + + 1. Copy the cert into `/etc/gitlab/trusted-certs`: + + ```shell + sudo cp cert.pem /etc/gitlab/trusted-certs/ + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. +1. Tail the logs to see the requests: + + ```shell + sudo gitlab-ctl tail gitaly + ``` + +NOTE: **Note:** When you specify `https` in the `external_url`, as in the example +above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If +certificates are not present, NGINX will fail to start. See the +[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) +for more information. + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Configure Prometheus + +The Omnibus GitLab package can be used to configure a standalone Monitoring node +running [Prometheus](../monitoring/prometheus/index.md) and +[Grafana](../monitoring/performance/grafana_configuration.md): + +1. SSH into the Monitoring node. +1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + Do not complete any other steps on the download page. +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + external_url 'http://gitlab.example.com' + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Login form + grafana['disable_login_form'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = 'toomanysecrets' + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_exporter['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + puma['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + gitlab_exporter['enable'] = false + ``` + +1. Prometheus also needs some scrape configs to pull all the data from the various + nodes where we configured exporters. Assuming that your nodes' IPs are: + + ```plaintext + 1.1.1.1: postgres + 1.1.1.2: redis + 1.1.1.3: gitaly1 + 1.1.1.4: rails1 + 1.1.1.5: rails2 + ``` + + Add the following to `/etc/gitlab/gitlab.rb`: + + ```ruby + prometheus['scrape_configs'] = [ + { + 'job_name': 'postgres', + 'static_configs' => [ + 'targets' => ['1.1.1.1:9187'], + ], + }, + { + 'job_name': 'redis', + 'static_configs' => [ + 'targets' => ['1.1.1.2:9121'], + ], + }, + { + 'job_name': 'gitaly', + 'static_configs' => [ + 'targets' => ['1.1.1.3:9236'], + ], + }, + { + 'job_name': 'gitlab-nginx', + 'static_configs' => [ + 'targets' => ['1.1.1.4:8060', '1.1.1.5:8060'], + ], + }, + { + 'job_name': 'gitlab-workhorse', + 'static_configs' => [ + 'targets' => ['1.1.1.4:9229', '1.1.1.5:9229'], + ], + }, + { + 'job_name': 'gitlab-rails', + 'metrics_path': '/-/metrics', + 'static_configs' => [ + 'targets' => ['1.1.1.4:8080', '1.1.1.5:8080'], + ], + }, + { + 'job_name': 'gitlab-sidekiq', + 'static_configs' => [ + 'targets' => ['1.1.1.4:8082', '1.1.1.5:8082'], + ], + }, + { + 'job_name': 'node', + 'static_configs' => [ + 'targets' => ['1.1.1.1:9100', '1.1.1.2:9100', '1.1.1.3:9100', '1.1.1.4:9100', '1.1.1.5:9100'], + ], + }, + ] + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. In the GitLab UI, set `admin/application_settings/metrics_and_profiling` > Metrics - Grafana to `/-/grafana` to +`http[s]://<MONITOR NODE>/-/grafana` + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> + +## Troubleshooting + +See the [troubleshooting documentation](troubleshooting.md). + +<div align="right"> + <a type="button" class="btn btn-default" href="#setup-components"> + Back to setup components <i class="fa fa-angle-double-up" aria-hidden="true"></i> + </a> +</div> diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index bd429fbc4b4..efeed3e9ffd 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -14,20 +14,20 @@ following the [2,000-user reference architecture](2k_users.md). > - **High Availability:** True > - **Test RPS rates:** API: 60 RPS, Web: 6 RPS, Git: 6 RPS -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|------------------------| -| GitLab Rails ([1](#footnotes)) | 3 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | -| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | +|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|----------------| +| GitLab Rails ([1](#footnotes)) | 3 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | +| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | +| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | | Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | ## Footnotes @@ -59,7 +59,7 @@ following the [2,000-user reference architecture](2k_users.md). 1. NFS can be used as an alternative for both repository data (replacing Gitaly) and object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). 1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets @@ -77,6 +77,3 @@ following the [2,000-user reference architecture](2k_users.md). or higher, are required for your CPU or Node counts accordingly. For more information, a [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 67f773a021f..dd94f5470b4 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -8,23 +8,23 @@ For a full list of reference architectures, see > - **High Availability:** True > - **Test RPS rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|------------------------| -| GitLab Rails ([1](#footnotes)) | 12 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | F32s v2 | -| PostgreSQL | 3 | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | D16s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 64 vCPU, 240GB Memory | n1-standard-64 | m5.16xlarge | D64s v3 | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | B1MS | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | D4s v3 | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | +|--------------------------------------------------------------|-------|---------------------------------|----------------|-----------------------|----------------| +| GitLab Rails ([1](#footnotes)) | 12 | 32 vCPU, 28.8GB Memory | `n1-highcpu-32` | `c5.9xlarge` | F32s v2 | +| PostgreSQL | 3 | 16 vCPU, 60GB Memory | `n1-standard-16` | `m5.4xlarge` | D16s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 64 vCPU, 240GB Memory | `n1-standard-64` | `m5.16xlarge` | D64s v3 | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | `g1-small` | `t2.small` | B1MS | +| Consul | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | `n1-standard-4` | `m5.xlarge` | D4s v3 | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | | Object Storage ([4](#footnotes)) | - | - | - | - | - | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | F8s v2 | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | `n1-highcpu-8` | `c5.2xlarge` | F8s v2 | ## Footnotes @@ -56,7 +56,7 @@ For a full list of reference architectures, see 1. NFS can be used as an alternative for both repository data (replacing Gitaly) and object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). 1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets @@ -74,6 +74,3 @@ For a full list of reference architectures, see or higher, are required for your CPU or Node counts accordingly. For more information, a [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 41ef6f369c2..604572b083e 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -8,20 +8,20 @@ For a full list of reference architectures, see > - **High Availability:** True > - **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS -| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS ([9](#footnotes)) | Azure([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|------------------------| -| GitLab Rails ([1](#footnotes)) | 3 | 16 vCPU, 14.4GB Memory | n1-highcpu-16 | c5.4xlarge | F16s v2 | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | D8s v3 | -| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | -| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | D2s v3 | +| Service | Nodes | Configuration ([8](#footnotes)) | GCP | AWS | Azure | +|--------------------------------------------------------------|-------|---------------------------------|---------------|-----------------------|----------------| +| GitLab Rails ([1](#footnotes)) | 3 | 16 vCPU, 14.4GB Memory | `n1-highcpu-16` | `c5.4xlarge` | F16s v2 | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 8 vCPU, 30GB Memory | `n1-standard-8` | `m5.2xlarge` | D8s v3 | +| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | +| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | `n1-standard-2` | `m5.large` | D2s v3 | | Object Storage ([4](#footnotes)) | - | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | F4s v2 | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | F2s v2 | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | `n1-highcpu-4` | `c5.xlarge` | F4s v2 | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | `n1-highcpu-2` | `c5.large` | F2s v2 | ## Footnotes @@ -53,7 +53,7 @@ For a full list of reference architectures, see 1. NFS can be used as an alternative for both repository data (replacing Gitaly) and object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). 1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets @@ -71,6 +71,3 @@ For a full list of reference architectures, see or higher, are required for your CPU or Node counts accordingly. For more information, a [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 26244368234..623d7f3f776 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -126,8 +126,8 @@ As long as at least one of each component is online and capable of handling the By adding automatic failover for database systems, you can enable higher uptime with additional database nodes. This extends the default database with cluster management and failover policies. -[PgBouncer](../../development/architecture.md#pgbouncer) in conjunction with -[Repmgr](../high_availability/database.md) is recommended. +[PgBouncer in conjunction with Repmgr](../postgresql/replication_and_failover.md) +is recommended. ### Instance level replication with GitLab Geo **(PREMIUM ONLY)** @@ -141,6 +141,9 @@ that can also be promoted in case of disaster. ## Configure GitLab to scale +NOTE: **Note:** +From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab 14.0, support for NFS for Git repositories is scheduled to be removed. Upgrade to [Gitaly Cluster](../gitaly/praefect.md) as soon as possible. + The following components are the ones you need to configure in order to scale GitLab. They are listed in the order you'll typically configure them if they are required by your [reference architecture](#reference-architectures) of choice. @@ -160,14 +163,33 @@ column. | [Consul](../../development/architecture.md#consul) ([3](#footnotes)) | Service discovery and health checks/failover | [Consul configuration](../high_availability/consul.md) **(PREMIUM ONLY)** | Yes | | [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | Yes | | [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | Yes | -| Repmgr | PostgreSQL cluster management and failover | [PostgreSQL and Repmgr configuration](../high_availability/database.md) | Yes | +| Repmgr | PostgreSQL cluster management and failover | [PostgreSQL and Repmgr configuration](../postgresql/replication_and_failover.md) | Yes | | [Redis](../../development/architecture.md#redis) ([3](#footnotes)) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | Yes | | Redis Sentinel | Redis | [Redis Sentinel configuration](../high_availability/redis.md) | Yes | -| [Gitaly](../../development/architecture.md#gitaly) ([2](#footnotes)) ([7](#footnotes)) ([10](#footnotes)) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#running-gitaly-on-its-own-server) | Yes | +| [Gitaly](../../development/architecture.md#gitaly) ([2](#footnotes)) ([7](#footnotes)) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#run-gitaly-on-its-own-server) | Yes | | [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | Yes | | [GitLab application services](../../development/architecture.md#unicorn)([1](#footnotes)) | Puma/Unicorn, Workhorse, GitLab Shell - serves front-end requests (UI, API, Git over HTTP/SSH) | [GitLab app scaling configuration](../high_availability/gitlab.md) | Yes | | [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling](../high_availability/monitoring_node.md) | Yes | +### Configuring select components with Cloud Native Helm + +We also provide [Helm charts](https://docs.gitlab.com/charts/) as a Cloud Native installation +method for GitLab. For the reference architectures, select components can be set up in this +way as an alternative if so desired. + +For these kind of setups we support using the charts in an [advanced configuration](https://docs.gitlab.com/charts/#advanced-configuration) +where stateful backend components, such as the database or Gitaly, are run externally - either +via Omnibus or reputable third party services. Note that we don't currently support running the +stateful components via Helm _at large scales_. + +When designing these environments you should refer to the respective [Reference Architecture](#available-reference-architectures) +above for guidance on sizing. Components run via Helm would be similarly scaled to their Omnibus +specs, only translated into Kubernetes resources. + +For example, if you were to set up a 50k installation with the Rails nodes being run in Helm, +then the same amount of resources as given for Omnibus should be given to the Kubernetes +cluster with the Rails nodes broken down into a number of smaller Pods across that cluster. + ## Footnotes 1. In our architectures we run each GitLab Rails node using the Puma webserver @@ -177,9 +199,7 @@ column. on workload. 1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend two nodes as an absolute minimum, - and at least four nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data + projects and their sizes. We recommend that each Gitaly node should store no more than 5TB of data and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) set to 20% of available CPUs. Additional nodes should be considered in conjunction with a review of expected data size and spread based on the recommendations above. @@ -198,7 +218,7 @@ column. 1. NFS can be used as an alternative for object storage but this isn't typically recommended for performance reasons. Note however it is required for [GitLab - Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). 1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets @@ -216,10 +236,3 @@ column. or higher, are required for your CPU or Node counts accordingly. For more information, a [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent and Azure-equivalent configurations are rough suggestions - and may change in the future. They have not yet been tested and validated. - -1. From GitLab 13.0, using NFS for Git repositories is deprecated. In GitLab - 14.0, support for NFS for Git repositories is scheduled to be removed. - Upgrade to [Gitaly Cluster](../gitaly/praefect.md) as soon as possible. diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md new file mode 100644 index 00000000000..15e377fe183 --- /dev/null +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -0,0 +1,329 @@ +# Troubleshooting a reference architecture set up + +This page serves as the troubleshooting documentation if you followed one of +the [reference architectures](index.md#reference-architectures). + +## Troubleshooting object storage + +### S3 API compatibility issues + +Not all S3 providers [are fully compatible](../../raketasks/backup_restore.md#other-s3-providers) +with the Fog library that GitLab uses. Symptoms include: + +```plaintext +411 Length Required +``` + +### GitLab Pages requires NFS + +If you intend to use [GitLab Pages](../../user/project/pages/index.md), this currently requires +[NFS](../high_availability/nfs.md). There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) +to remove this dependency. In the future, GitLab Pages may use +[object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/208135). + +The dependency on disk storage also prevents Pages being deployed using the +[GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/37). + +### Incremental logging is required for CI to use object storage + +If you configure GitLab to use object storage for CI logs and artifacts, +[you must also enable incremental logging](../job_logs.md#new-incremental-logging-architecture). + +### Proxy Download + +A number of the use cases for object storage allow client traffic to be redirected to the +object storage back end, like when Git clients request large files via LFS or when +downloading CI artifacts and logs. + +When the files are stored on local block storage or NFS, GitLab has to act as a proxy. +With object storage, the default behavior is for GitLab to redirect to the object +storage device rather than proxy the request. + +The `proxy_download` setting controls this behavior: the default is generally `false`. +Verify this in the documentation for each use case. Set it to `true` to make +GitLab proxy the files rather than redirect. + +When not proxying files, GitLab returns an +[HTTP 302 redirect with a pre-signed, time-limited object storage URL](https://gitlab.com/gitlab-org/gitlab/-/issues/32117#note_218532298). +This can result in some of the following problems: + +- If GitLab is using non-secure HTTP to access the object storage, clients may generate +`https->http` downgrade errors and refuse to process the redirect. The solution to this +is for GitLab to use HTTPS. LFS, for example, will generate this error: + + ```plaintext + LFS: lfsapi/client: refusing insecure redirect, https->http + ``` + +- Clients will need to trust the certificate authority that issued the object storage +certificate, or may return common TLS errors such as: + + ```plaintext + x509: certificate signed by unknown authority + ``` + +- Clients will need network access to the object storage. Errors that might result +if this access is not in place include: + + ```plaintext + Received status code 403 from server: Forbidden + ``` + +### ETag mismatch + +Using the default GitLab settings, some object storage back-ends such as +[MinIO](https://gitlab.com/gitlab-org/gitlab/-/issues/23188) +and [Alibaba](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564) +might generate `ETag mismatch` errors. + +When using GitLab direct upload, the +[workaround for MinIO](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564#note_244497658) +is to use the `--compat` parameter on the server. + +We are working on a fix to GitLab component Workhorse, and also +a workaround, in the mean time, to +[allow ETag verification to be disabled](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18175). + +## Troubleshooting Redis + +If the application node cannot connect to the Redis node, check your firewall rules and +make sure Redis can accept TCP connections under port `6379`. + +## Troubleshooting Gitaly + +### Checking versions when using standalone Gitaly nodes + +When using standalone Gitaly nodes, you must make sure they are the same version +as GitLab to ensure full compatibility. Check **Admin Area > Gitaly Servers** on +your GitLab instance and confirm all Gitaly Servers are `Up to date`. + + + +### `gitaly-debug` + +The `gitaly-debug` command provides "production debugging" tools for Gitaly and Git +performance. It is intended to help production engineers and support +engineers investigate Gitaly performance problems. + +If you're using GitLab 11.6 or newer, this tool should be installed on +your GitLab / Gitaly server already at `/opt/gitlab/embedded/bin/gitaly-debug`. +If you're investigating an older GitLab version you can compile this +tool offline and copy the executable to your server: + +```shell +git clone https://gitlab.com/gitlab-org/gitaly.git +cd cmd/gitaly-debug +GOOS=linux GOARCH=amd64 go build -o gitaly-debug +``` + +To see the help page of `gitaly-debug` for a list of supported sub-commands, run: + +```shell +gitaly-debug -h +``` + +### Commits, pushes, and clones return a 401 + +```plaintext +remote: GitLab: 401 Unauthorized +``` + +You will need to sync your `gitlab-secrets.json` file with your GitLab +app nodes. + +### Client side gRPC logs + +Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC +client has its own log file which may contain useful information when +you are seeing Gitaly errors. You can control the log level of the +gRPC client with the `GRPC_LOG_LEVEL` environment variable. The +default level is `WARN`. + +You can run a gRPC trace with: + +```shell +sudo GRPC_TRACE=all GRPC_VERBOSITY=DEBUG gitlab-rake gitlab:gitaly:check +``` + +### Observing `gitaly-ruby` traffic + +[`gitaly-ruby`](../gitaly/index.md#gitaly-ruby) is an internal implementation detail of Gitaly, +so, there's not that much visibility into what goes on inside +`gitaly-ruby` processes. + +If you have Prometheus set up to scrape your Gitaly process, you can see +request rates and error codes for individual RPCs in `gitaly-ruby` by +querying `grpc_client_handled_total`. Strictly speaking, this metric does +not differentiate between `gitaly-ruby` and other RPCs, but in practice +(as of GitLab 11.9), all gRPC calls made by Gitaly itself are internal +calls from the main Gitaly process to one of its `gitaly-ruby` sidecars. + +Assuming your `grpc_client_handled_total` counter only observes Gitaly, +the following query shows you RPCs are (most likely) internally +implemented as calls to `gitaly-ruby`: + +```prometheus +sum(rate(grpc_client_handled_total[5m])) by (grpc_method) > 0 +``` + +### Repository changes fail with a `401 Unauthorized` error + +If you're running Gitaly on its own server and notice that users can +successfully clone and fetch repositories (via both SSH and HTTPS), but can't +push to them or make changes to the repository in the web UI without getting a +`401 Unauthorized` message, then it's possible Gitaly is failing to authenticate +with the other nodes due to having the wrong secrets file. + +Confirm the following are all true: + +- When any user performs a `git push` to any repository on this Gitaly node, it + fails with the following error (note the `401 Unauthorized`): + + ```shell + remote: GitLab: 401 Unauthorized + To <REMOTE_URL> + ! [remote rejected] branch-name -> branch-name (pre-receive hook declined) + error: failed to push some refs to '<REMOTE_URL>' + ``` + +- When any user adds or modifies a file from the repository using the GitLab + UI, it immediately fails with a red `401 Unauthorized` banner. +- Creating a new project and [initializing it with a README](../../gitlab-basics/create-project.md#blank-projects) + successfully creates the project but doesn't create the README. +- When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.html#tail-logs-in-a-console-on-the-server) on an app node and reproducing the error, you get `401` errors + when reaching the `/api/v4/internal/allowed` endpoint: + + ```shell + # api_json.log + { + "time": "2019-07-18T00:30:14.967Z", + "severity": "INFO", + "duration": 0.57, + "db": 0, + "view": 0.57, + "status": 401, + "method": "POST", + "path": "\/api\/v4\/internal\/allowed", + "params": [ + { + "key": "action", + "value": "git-receive-pack" + }, + { + "key": "changes", + "value": "REDACTED" + }, + { + "key": "gl_repository", + "value": "REDACTED" + }, + { + "key": "project", + "value": "\/path\/to\/project.git" + }, + { + "key": "protocol", + "value": "web" + }, + { + "key": "env", + "value": "{\"GIT_ALTERNATE_OBJECT_DIRECTORIES\":[],\"GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE\":[],\"GIT_OBJECT_DIRECTORY\":null,\"GIT_OBJECT_DIRECTORY_RELATIVE\":null}" + }, + { + "key": "user_id", + "value": "2" + }, + { + "key": "secret_token", + "value": "[FILTERED]" + } + ], + "host": "gitlab.example.com", + "ip": "REDACTED", + "ua": "Ruby", + "route": "\/api\/:version\/internal\/allowed", + "queue_duration": 4.24, + "gitaly_calls": 0, + "gitaly_duration": 0, + "correlation_id": "XPUZqTukaP3" + } + + # nginx_access.log + [IP] - - [18/Jul/2019:00:30:14 +0000] "POST /api/v4/internal/allowed HTTP/1.1" 401 30 "" "Ruby" + ``` + +To fix this problem, confirm that your `gitlab-secrets.json` file +on the Gitaly node matches the one on all other nodes. If it doesn't match, +update the secrets file on the Gitaly node to match the others, then +[reconfigure the node](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### Command line tools cannot connect to Gitaly + +If you are having trouble connecting to a Gitaly node with command line (CLI) tools, and certain actions result in a `14: Connect Failed` error message, it means that gRPC cannot reach your Gitaly node. + +Verify that you can reach Gitaly via TCP: + +```shell +sudo gitlab-rake gitlab:tcp_check[GITALY_SERVER_IP,GITALY_LISTEN_PORT] +``` + +If the TCP connection fails, check your network settings and your firewall rules. If the TCP connection succeeds, your networking and firewall rules are correct. + +If you use proxy servers in your command line environment, such as Bash, these can interfere with your gRPC traffic. + +If you use Bash or a compatible command line environment, run the following commands to determine whether you have proxy servers configured: + +```shell +echo $http_proxy +echo $https_proxy +``` + +If either of these variables have a value, your Gitaly CLI connections may be getting routed through a proxy which cannot connect to Gitaly. + +To remove the proxy setting, run the following commands (depending on which variables had values): + +```shell +unset http_proxy +unset https_proxy +``` + +### Gitaly not listening on new address after reconfiguring + +When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. + +When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/issues/2521) is resolved. + +### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly node + +If this error occurs even though file permissions are correct, it's likely that +the Gitaly node is experiencing +[clock drift](https://en.wikipedia.org/wiki/Clock_drift). + +Please ensure that the GitLab and Gitaly nodes are synchronized and use an NTP time +server to keep them synchronized if possible. + +## Troubleshooting the GitLab Rails application + +- `mount: wrong fs type, bad option, bad superblock on` + +You have not installed the necessary NFS client utilities. See step 1 above. + +- `mount: mount point /var/opt/gitlab/... does not exist` + +This particular directory does not exist on the NFS server. Ensure +the share is exported and exists on the NFS server and try to remount. + +## Troubleshooting Monitoring + +If the monitoring node is not receiving any data, check that the exporters are +capturing data. + +```shell +curl http[s]://localhost:<EXPORTER LISTENING PORT>/metric +``` + +or + +```shell +curl http[s]://localhost:<EXPORTER LISTENING PORT>/-/metric +``` diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index 283401dafff..87f901becf5 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -10,7 +10,7 @@ storage shards) to distribute the storage load between several mount points. > - You must have at least one storage path called `default`. > - The paths are defined in key-value pairs. The key is an arbitrary name you > can pick to name the file path. -> - The target directories and any of its subpaths must not be a symlink. +> - The target directories and any of its sub-paths must not be a symlink. > - No target directory may be a sub-directory of another; no nesting. Example: this is OK: @@ -57,7 +57,7 @@ storage2: Now that you've read that big fat warning above, let's edit the configuration files and add the full paths of the alternative repository storage paths. In -the example below, we add two more mountpoints that are named `nfs_1` and `nfs_2` +the example below, we add two more mount points that are named `nfs_1` and `nfs_2` respectively. NOTE: **Note:** This example uses NFS. We do not recommend using EFS for storage as it may impact GitLab's performance. See the [relevant documentation](high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index a95178c01e2..825da08b66e 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -1,6 +1,6 @@ # Repository storage types **(CORE ONLY)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28283) in GitLab 10.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28283) in GitLab 10.0. > - Hashed storage became the default for new installations in GitLab 12.0 > - Hashed storage is enabled by default for new and renamed projects in GitLab 13.0. @@ -108,7 +108,7 @@ The output includes the project ID and the project name: ### Hashed object pools -> [Introduced](https://gitlab.com/gitlab-org/gitaly/issues/1606) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/1606) in GitLab 12.1. DANGER: **Danger:** Do not run `git prune` or `git gc` in pool repositories! This can diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 0b8c66805ae..6df0f187a42 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto disqus_identifier: 'https://docs.gitlab.com/ee/administration/custom_hooks.html' --- @@ -7,7 +10,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/administration/custom_hooks.html' > **Notes:** > -> - Server hooks were [introduced](https://gitlab.com/gitlab-org/gitlab/issues/196051) in GitLab 12.8 replacing Custom Hooks. +> - Server hooks were [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196051) in GitLab 12.8 replacing Custom Hooks. > - Server hooks must be configured on the filesystem of the GitLab server. Only GitLab server administrators will be able to complete these tasks. Please explore [webhooks](../user/project/integrations/webhooks.md) and [GitLab CI/CD](../ci/README.md) as an option if you do not have filesystem access. For a user-configurable Git hook interface, see [Push Rules](../push_rules/push_rules.md), available in GitLab Starter **(STARTER)**. > - Server hooks won't be replicated to secondary nodes if you use [GitLab Geo](geo/replication/index.md). @@ -65,9 +68,18 @@ Follow the steps below to properly set up a server hook for all repositories: `/home/git/gitlab-shell/hooks`. For Omnibus installs the path is usually `/opt/gitlab/embedded/service/gitlab-shell/hooks`. To look in a different directory for the global custom hooks, - set `custom_hooks_dir` in the GitLab Shell config. For - Omnibus installations, this can be set in `gitlab.rb`; and in source - installations, this can be set in `gitlab-shell/config.yml`. + set `custom_hooks_dir` in the Gitaly config. For Omnibus installations, this is set + in `gitlab.rb`. For source installations, the configuration location depends on the + GitLab version. For: + + - GitLab 13.0 and earlier, this is set in `gitlab-shell/config.yml`. + - GitLab 13.1 and later, this is set in `gitaly/config.toml` under the `[hooks]` + section. + + NOTE: **Note:** + The `custom_hooks_dir` value in `gitlab-shell/config.yml` is still honored in GitLab + 13.1 and later if the value in `gitaly/config.toml` is blank or non-existent. + 1. Create a new directory in this location. Depending on your hook, it will be either a `pre-receive.d`, `post-receive.d`, or `update.d` directory. 1. Inside this new directory, add your hook. Hooks can be diff --git a/doc/administration/snippets/index.md b/doc/administration/snippets/index.md index e6bbfa8cf00..cf3d8bec1a6 100644 --- a/doc/administration/snippets/index.md +++ b/doc/administration/snippets/index.md @@ -8,7 +8,7 @@ Adjust the snippets' settings of your GitLab instance. ## Snippets content size limit -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31133) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31133) in GitLab 12.6. You can set a content size max limit in snippets. This limit can prevent abuses of the feature. The default content size limit is **52428800 Bytes** (50MB). diff --git a/doc/administration/timezone.md b/doc/administration/timezone.md index d6112c45141..3cfee8630b7 100644 --- a/doc/administration/timezone.md +++ b/doc/administration/timezone.md @@ -15,7 +15,7 @@ To see all available time zones, run `bundle exec rake time:zones:all`. For Omnibus installations, run `gitlab-rake time:zones:all`. NOTE: **Note:** -Currently, this Rake task does not list timezones in TZInfo format required by Omnibus GitLab during a reconfigure: [#58672](https://gitlab.com/gitlab-org/gitlab-foss/issues/58672). +Currently, this Rake task does not list timezones in TZInfo format required by Omnibus GitLab during a reconfigure: [#27209](https://gitlab.com/gitlab-org/gitlab/-/issues/27209). ## Changing time zone in Omnibus installations diff --git a/doc/administration/troubleshooting/debug.md b/doc/administration/troubleshooting/debug.md index 1e1b2ad8378..55fd6462bc3 100644 --- a/doc/administration/troubleshooting/debug.md +++ b/doc/administration/troubleshooting/debug.md @@ -244,7 +244,7 @@ separate Rails process to debug the issue: For example: ```ruby - app.get 'https://gitlab.com/gitlab-org/gitlab-foss/issues/1?private_token=123456' + app.get 'https://gitlab.com/gitlab-org/gitlab-foss/-/issues/1?private_token=123456' ``` 1. In a new window, run `top`. It should show this Ruby process using 100% CPU. Write down the PID. diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index a39fe4ba8c3..12b82e4bc48 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -189,7 +189,7 @@ Moving past that, it is best to attempt the same search using the [Elasticsearch If the results: -- Sync up, then there is not a technical "issue" per se. Instead, it might be a problem +- Sync up, then there is not a technical "issue." Instead, it might be a problem with the Elasticsearch filters we are using. This can be complicated, so it is best to escalate to GitLab support to check these and guide you on the potential on whether or not a feature request is needed. @@ -330,10 +330,10 @@ feel free to update that page with issues you encounter and solutions. Setting up Elasticsearch isn't too bad, but it can be a bit finicky and time consuming. -The easiest method is to spin up a docker container with the required version and +The easiest method is to spin up a Docker container with the required version and bind ports 9200/9300 so it can be used. -The following is an example of running a docker container of Elasticsearch v7.2.0: +The following is an example of running a Docker container of Elasticsearch v7.2.0: ```shell docker pull docker.elastic.co/elasticsearch/elasticsearch:7.2.0 @@ -342,7 +342,7 @@ docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elas From here, you can: -- Grab the IP of the docker container (use `docker inspect <container_id>`) +- Grab the IP of the Docker container (use `docker inspect <container_id>`) - Use `<IP.add.re.ss:9200>` to communicate with it. This is a quick method to test out Elasticsearch, but by no means is this a diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 2cbc994fb4c..33af356b37d 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -263,7 +263,7 @@ p.import_state.mark_as_failed("Failed manually through console.") In a specific situation, an imported repository needed to be renamed. The Support Team was informed of a backup restore that failed on a single repository, which created -the project with an empty repository. The project was successfully restored to a dev +the project with an empty repository. The project was successfully restored to a development instance, then exported, and imported into a new project under a different name. The Support Team was able to transfer the incorrectly named imported project into the @@ -302,7 +302,7 @@ you will see two pushes with the same "from" SHA: ```ruby p = Project.find_with_namespace('u/p') -p.events.code_push.last(100).each do |e| +p.events.pushed_action.last(100).each do |e| printf "%-20.20s %8s...%8s (%s)\n", e.data[:ref], e.data[:before], e.data[:after], e.author.try(:username) end ``` @@ -311,7 +311,7 @@ GitLab 9.5 and above: ```ruby p = Project.find_by_full_path('u/p') -p.events.code_push.last(100).each do |e| +p.events.pushed_action.last(100).each do |e| printf "%-20.20s %8s...%8s (%s)\n", e.push_event_payload[:ref], e.push_event_payload[:commit_from], e.push_event_payload[:commit_to], e.author.try(:username) end ``` @@ -380,39 +380,6 @@ user = User.find_by_username '' user.skip_reconfirmation! ``` -### Get an admin token - -```ruby -# Get the first admin's first access token (no longer works on 11.9+. see: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22743) -User.where(admin:true).first.personal_access_tokens.first.token - -# Get the first admin's private token (no longer works on 10.2+) -User.where(admin:true).private_token -``` - -### Create personal access token - -```ruby -personal_access_token = User.find(123).personal_access_tokens.create( - name: 'apitoken', - impersonation: false, - scopes: [:api] -) - -puts personal_access_token.token -``` - -You might also want to manually set the token string: - -```ruby -User.find(123).personal_access_tokens.create( - name: 'apitoken', - token_digest: Gitlab::CryptoHelper.sha256('some-token-string-here'), - impersonation: false, - scopes: [:api] -) -``` - ### Active users & Historical users ```ruby @@ -518,7 +485,7 @@ group.project_creation_level=0 ### Remove redirecting routes -See <https://gitlab.com/gitlab-org/gitlab-foss/issues/41758#note_54828133>. +See <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41758#note_54828133>. ```ruby path = 'foo' @@ -576,7 +543,7 @@ This section has been moved to the [job artifacts troubleshooting documentation] ### Find reason failure (for when build trace is empty) (Introduced in 10.3.0) -See <https://gitlab.com/gitlab-org/gitlab-foss/issues/41111>. +See <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41111>. ```ruby build = Ci::Build.find(78420) @@ -620,10 +587,26 @@ Gitlab::CurrentSettings.current_application_settings.runners_registration_token ## License -### See license plan name (since v9.3.0-ee) +### See current license information ```ruby +# License information (name, company, email address) +License.current.licensee + +# Plan: License.current.plan + +# Uploaded: +License.current.created_at + +# Started: +License.current.starts_at + +# Expires at: +License.current.expires_at + +# Is this a trial license? +License.current.trial? ``` ### Check if a project feature is available on the instance @@ -636,7 +619,7 @@ License.current.feature_available?(:jira_dev_panel_integration) ### Check if a project feature is available in a project -Features listed in <https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/license.rb>. +Features listed in [`license.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/license.rb). ```ruby p = Project.find_by_full_path('<group>/<project>') diff --git a/doc/administration/troubleshooting/index.md b/doc/administration/troubleshooting/index.md new file mode 100644 index 00000000000..50f192b1983 --- /dev/null +++ b/doc/administration/troubleshooting/index.md @@ -0,0 +1,20 @@ +# Troubleshooting a GitLab installation + +Below are some resources to help you troubleshoot a GitLab installation +in case something goes wrong: + +- [Debugging tips](debug.md) +- [Diagnostics tools](diagnostics_tools.md) +- [Elasticsearch](elasticsearch.md) +- [GitLab Rails console cheat sheet](gitlab_rails_cheat_sheet.md) +- [Group SAML and SCIM troubleshooting](group_saml_scim.md) **(SILVER ONLY)** +- [Kubernetes cheat sheet](kubernetes_cheat_sheet.md) +- [Linux cheat sheet](linux_cheat_sheet.md) +- [Parsing GitLab logs with `jq`](log_parsing.md) +- [Navigating GitLab via Rails console](navigating_gitlab_via_rails_console.md) +- [PostgreSQL](postgresql.md) +- [Sidekiq](sidekiq.md) +- [SSL](ssl.md) + +If you need a testing environment to troubleshoot, see the +[apps for a testing environment](test_environments.md). diff --git a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md index cab073b9924..01532032b49 100644 --- a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md +++ b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md @@ -85,7 +85,7 @@ and they will assist you with any issues you are having. ## GitLab-specific Kubernetes information - Minimal config that can be used to test a Kubernetes Helm chart can be found - [here](https://gitlab.com/gitlab-org/charts/gitlab/issues/620). + [here](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/620). - Tailing logs of a separate pod. An example for a Webservice pod: @@ -186,7 +186,7 @@ and they will assist you with any issues you are having. helm upgrade <release name> <chart path> -f gitlab.yaml ``` - After <https://gitlab.com/gitlab-org/charts/gitlab/issues/780> is fixed, it should + After <https://gitlab.com/gitlab-org/charts/gitlab/-/issues/780> is fixed, it should be possible to use [Updating GitLab using the Helm Chart](https://docs.gitlab.com/charts/index.html#updating-gitlab-using-the-helm-chart) for upgrades. diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index 65a6bffca44..e5a4dffb3cc 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -31,7 +31,7 @@ This section is for links to information elsewhere in the GitLab documentation. - Destructively reseeding the GitLab database. - Guidance around updating packaged PostgreSQL, including how to stop it happening automatically. -- [More about external PostgreSQL](../external_database.md) +- [More about external PostgreSQL](../postgresql/external.md) - [Running Geo with external PostgreSQL](../geo/replication/external_database.md) @@ -45,8 +45,8 @@ This section is for links to information elsewhere in the GitLab documentation. - Managing Omnibus PostgreSQL versions [from the development docs](https://docs.gitlab.com/omnibus/development/managing-postgresql-versions.html) -- [PostgreSQL scaling](../high_availability/database.md) - - including [troubleshooting](../high_availability/database.md#troubleshooting) `gitlab-ctl repmgr-check-master` and PgBouncer errors +- [PostgreSQL scaling](../postgresql/replication_and_failover.md) + - including [troubleshooting](../postgresql/replication_and_failover.md#troubleshooting) `gitlab-ctl repmgr-check-master` and PgBouncer errors - [Developer database documentation](../../development/README.md#database-guides) - some of which is absolutely not for production use. Including: - understanding EXPLAIN plans @@ -55,8 +55,8 @@ This section is for links to information elsewhere in the GitLab documentation. - [GitLab database requirements](../../install/requirements.md#database) including - Support for MySQL was removed in GitLab 12.1; [migrate to PostgreSQL](../../update/mysql_to_postgresql.md) - - required extension pg_trgm - - required extension postgres_fdw for Geo + - required extension `pg_trgm` + - required extension `postgres_fdw` for Geo - Errors like this in the `production/sidekiq` log; see: [Set default_transaction_isolation into read committed](https://docs.gitlab.com/omnibus/settings/database.html#set-default_transaction_isolation-into-read-committed): @@ -95,15 +95,15 @@ This section is for links to information elsewhere in the GitLab documentation. References: -- [Issue #1 Deadlocks with GitLab 12.1, PostgreSQL 10.7](https://gitlab.com/gitlab-org/gitlab/issues/30528) +- [Issue #1 Deadlocks with GitLab 12.1, PostgreSQL 10.7](https://gitlab.com/gitlab-org/gitlab/-/issues/30528) - [Customer ticket (internal) GitLab 12.1.6](https://gitlab.zendesk.com/agent/tickets/134307) and [Google doc (internal)](https://docs.google.com/document/d/19xw2d_D1ChLiU-MO1QzWab-4-QXgsIUcN5e_04WTKy4) -- [Issue #2 deadlocks can occur if an instance is flooded with pushes](https://gitlab.com/gitlab-org/gitlab/issues/33650). Provided for context about how GitLab code can have this sort of unanticipated effect in unusual situations. +- [Issue #2 deadlocks can occur if an instance is flooded with pushes](https://gitlab.com/gitlab-org/gitlab/-/issues/33650). Provided for context about how GitLab code can have this sort of unanticipated effect in unusual situations. ```plaintext ERROR: deadlock detected ``` -Three applicable timeouts are identified in the issue [#1](https://gitlab.com/gitlab-org/gitlab/issues/30528); our recommended settings are as follows: +Three applicable timeouts are identified in the issue [#1](https://gitlab.com/gitlab-org/gitlab/-/issues/30528); our recommended settings are as follows: ```ini deadlock_timeout = 5s @@ -111,20 +111,20 @@ statement_timeout = 15s idle_in_transaction_session_timeout = 60s ``` -Quoting from issue [#1](https://gitlab.com/gitlab-org/gitlab/issues/30528): +Quoting from issue [#1](https://gitlab.com/gitlab-org/gitlab/-/issues/30528): > "If a deadlock is hit, and we resolve it through aborting the transaction after a short period, then the retry mechanisms we already have will make the deadlocked piece of work try again, and it's unlikely we'll deadlock multiple times in a row." TIP: **Tip:** In support, our general approach to reconfiguring timeouts (applies also to the HTTP stack as well) is that it's acceptable to do it temporarily as a workaround. If it makes GitLab usable for the customer, then it buys time to understand the problem more completely, implement a hot fix, or make some other change that addresses the root cause. Generally, the timeouts should be put back to reasonable defaults once the root cause is resolved. -In this case, the guidance we had from development was to drop deadlock_timeout and/or statement_timeout but to leave the third setting at 60s. Setting idle_in_transaction protects the database from sessions potentially hanging for days. There's more discussion in [the issue relating to introducing this timeout on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/issues/1053). +In this case, the guidance we had from development was to drop deadlock_timeout and/or statement_timeout but to leave the third setting at 60s. Setting idle_in_transaction protects the database from sessions potentially hanging for days. There's more discussion in [the issue relating to introducing this timeout on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/1053). PostgresSQL defaults: - `statement_timeout = 0` (never) - `idle_in_transaction_session_timeout = 0` (never) -Comments in issue [#1](https://gitlab.com/gitlab-org/gitlab/issues/30528) indicate that these should both be set to at least a number of minutes for all Omnibus installations (so they don't hang indefinitely). However, 15s for statement_timeout is very short, and will only be effective if the underlying infrastructure is very performant. +Comments in issue [#1](https://gitlab.com/gitlab-org/gitlab/-/issues/30528) indicate that these should both be set to at least a number of minutes for all Omnibus installations (so they don't hang indefinitely). However, 15s for statement_timeout is very short, and will only be effective if the underlying infrastructure is very performant. See current settings with: diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index ca21c038267..5109a3baff2 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -29,9 +29,18 @@ Example: gitlab_rails['env'] = {"SIDEKIQ_LOG_ARGUMENTS" => "1"} ``` -Please note: It is not recommend to enable this setting in production because some -Sidekiq jobs (such as sending a password reset email) take secret arguments (for -example the password reset token). +This does not log all job arguments. To avoid logging sensitive +information (for instance, password reset tokens), it logs numeric +arguments for all workers, with overrides for some specific workers +where their arguments are not sensitive. + +Example log output: + +```json +{"severity":"INFO","time":"2020-06-08T14:37:37.892Z","class":"AdminEmailsWorker","args":["[FILTERED]","[FILTERED]","[FILTERED]"],"retry":3,"queue":"admin_emails","backtrace":true,"jid":"9e35e2674ac7b12d123e13cc","created_at":"2020-06-08T14:37:37.373Z","meta.user":"root","meta.caller_id":"Admin::EmailsController#create","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:6dc94409cfdd4d77:9fbe19bdee865293:1","enqueued_at":"2020-06-08T14:37:37.410Z","pid":65011,"message":"AdminEmailsWorker JID-9e35e2674ac7b12d123e13cc: done: 0.48085 sec","job_status":"done","scheduling_latency_s":0.001012,"redis_calls":9,"redis_duration_s":0.004608,"redis_read_bytes":696,"redis_write_bytes":6141,"duration_s":0.48085,"cpu_s":0.308849,"completed_at":"2020-06-08T14:37:37.892Z","db_duration_s":0.010742} +{"severity":"INFO","time":"2020-06-08T14:37:37.894Z","class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ActionMailer::DeliveryJob","queue":"mailers","args":["[FILTERED]"],"retry":3,"backtrace":true,"jid":"e47a4f6793d475378432e3c8","created_at":"2020-06-08T14:37:37.884Z","meta.user":"root","meta.caller_id":"AdminEmailsWorker","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:29344de0f966446d:5c3b0e0e1bef987b:1","enqueued_at":"2020-06-08T14:37:37.885Z","pid":65011,"message":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper JID-e47a4f6793d475378432e3c8: start","job_status":"start","scheduling_latency_s":0.009473} +{"severity":"INFO","time":"2020-06-08T14:39:50.648Z","class":"NewIssueWorker","args":["455","1"],"retry":3,"queue":"new_issue","backtrace":true,"jid":"a24af71f96fd129ec47f5d1e","created_at":"2020-06-08T14:39:50.643Z","meta.user":"root","meta.project":"h5bp/html5-boilerplate","meta.root_namespace":"h5bp","meta.caller_id":"Projects::IssuesController#create","correlation_id":"f9UCZHqhuP7","uber-trace-id":"28f65730f99f55a3:a5d2b62dec38dffc:48ddd092707fa1b7:1","enqueued_at":"2020-06-08T14:39:50.646Z","pid":65011,"message":"NewIssueWorker JID-a24af71f96fd129ec47f5d1e: start","job_status":"start","scheduling_latency_s":0.001144} +``` When using [Sidekiq JSON logging](../logs.md#sidekiqlog), arguments logs are limited to a maximum size of 10 kilobytes of text; @@ -86,13 +95,13 @@ sudo apt-get install linux-tools-common linux-tools-generic linux-tools-`uname - sudo yum install perf ``` -Run perf against the Sidekiq PID: +Run `perf` against the Sidekiq PID: ```shell sudo perf record -p <sidekiq_pid> ``` -Let this run for 30-60 seconds and then press Ctrl-C. Then view the perf report: +Let this run for 30-60 seconds and then press Ctrl-C. Then view the `perf` report: ```shell $ sudo perf report @@ -105,13 +114,13 @@ Samples: 348K of event 'cycles', Event count (approx.): 280908431073 0.10% ruby libc-2.12.so [.] _int_free ``` -Above you see sample output from a perf report. It shows that 97% of the CPU is +Above you see sample output from a `perf` report. It shows that 97% of the CPU is being spent inside Nokogiri and `xmlXPathNodeSetMergeAndClear`. For something this obvious you should then go investigate what job in GitLab would use Nokogiri and XPath. Combine with `TTIN` or `gdb` output to show the corresponding Ruby code where this is happening. -## The GNU Project Debugger (gdb) +## The GNU Project Debugger (`gdb`) `gdb` can be another effective tool for debugging Sidekiq. It gives you a little more interactive way to look at each thread and see what's causing problems. diff --git a/doc/administration/troubleshooting/test_environments.md b/doc/administration/troubleshooting/test_environments.md index e9db5f64446..80ccd15aa22 100644 --- a/doc/administration/troubleshooting/test_environments.md +++ b/doc/administration/troubleshooting/test_environments.md @@ -16,7 +16,7 @@ are only available internally at GitLab. ## Docker -The following were tested on docker containers running in the cloud. Support Engineers, +The following were tested on Docker containers running in the cloud. Support Engineers, please see [these docs](https://gitlab.com/gitlab-com/dev-resources/tree/master/dev-resources#running-docker-containers) on how to run Docker containers on `dev-resources`. Other setups haven't been tested, but contributions are welcome. diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index 0a300084342..620f349912c 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -168,8 +168,8 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | `provider` | Always `OpenStack` for compatible hosts | `OpenStack` | | `openstack_username` | OpenStack username | | | `openstack_api_key` | OpenStack API key | | -| `openstack_temp_url_key` | OpenStack key for generating temporary urls | | -| `openstack_auth_url` | OpenStack authentication endpont | | +| `openstack_temp_url_key` | OpenStack key for generating temporary URLs | | +| `openstack_auth_url` | OpenStack authentication endpoint | | | `openstack_region` | OpenStack region | | | `openstack_tenant` | OpenStack tenant ID | diff --git a/doc/api/README.md b/doc/api/README.md index 34d496a37fe..6cbb99a76cb 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -53,12 +53,12 @@ between v3 and v4; please read the [v3 to v4 documentation](v3_to_v4.md) ### Current status Currently only API version v4 is available. Version v3 was removed in -[GitLab 11.0](https://gitlab.com/gitlab-org/gitlab-foss/issues/36819). +[GitLab 11.0](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/36819). ## Basic usage API requests should be prefixed with `api` and the API version. The API version -is defined in [`lib/api.rb`](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/lib/api/api.rb). For example, the root of the v4 API +is defined in [`lib/api.rb`](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/api/api.rb). For example, the root of the v4 API is at `/api/v4`. Example of a valid API request using cURL: @@ -107,13 +107,13 @@ You can use an [OAuth2 token](oauth2.md) to authenticate with the API by passing Example of using the OAuth2 token in a parameter: ```shell -curl https://gitlab.example.com/api/v4/projects?access_token=OAUTH-TOKEN +curl "https://gitlab.example.com/api/v4/projects?access_token=OAUTH-TOKEN" ``` Example of using the OAuth2 token in a header: ```shell -curl --header "Authorization: Bearer OAUTH-TOKEN" https://gitlab.example.com/api/v4/projects +curl --header "Authorization: Bearer OAUTH-TOKEN" "https://gitlab.example.com/api/v4/projects" ``` Read more about [GitLab as an OAuth2 provider](oauth2.md). @@ -126,19 +126,19 @@ or the `Private-Token` header. Example of using the personal/project access token in a parameter: ```shell -curl https://gitlab.example.com/api/v4/projects?private_token=<your_access_token> +curl "https://gitlab.example.com/api/v4/projects?private_token=<your_access_token>" ``` Example of using the personal/project access token in a header: ```shell -curl --header "Private-Token: <your_access_token>" https://gitlab.example.com/api/v4/projects +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects" ``` You can also use personal/project access tokens with OAuth-compliant headers: ```shell -curl --header "Authorization: Bearer <your_access_token>" https://gitlab.example.com/api/v4/projects +curl --header "Authorization: Bearer <your_access_token>" "https://gitlab.example.com/api/v4/projects" ``` ### Session cookie @@ -180,7 +180,7 @@ Impersonation tokens are used exactly like regular personal access tokens, and c #### Disable impersonation -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/40385) in GitLab 11.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/40385) in GitLab 11.6. By default, impersonation is enabled. To disable impersonation: @@ -348,7 +348,7 @@ and we request the second page (`page=2`) of [comments](notes.md) of the issue with ID `8` which belongs to the project with ID `8`: ```shell -curl --head --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/8/issues/8/notes?per_page=3&page=2 +curl --head --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?per_page=3&page=2" ``` The response will then be: @@ -417,10 +417,14 @@ The response header includes a link to the next page. For example: HTTP/1.1 200 OK ... Links: <https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc&id_after=42>; rel="next" +Link: <https://gitlab.example.com/api/v4/projects?pagination=keyset&per_page=50&order_by=id&sort=asc&id_after=42>; rel="next" Status: 200 OK ... ``` +CAUTION: **Deprecation:** +The `Links` Header will be removed in GitLab 14.0 to be aligned with the [W3C specification](https://www.w3.org/wiki/LinkHeader) + The link to the next page contains an additional filter `id_after=42` which excludes records we have retrieved already. Note the type of filter depends on the `order_by` option used and we may have more than one additional filter. @@ -450,7 +454,7 @@ The `:id` path parameter needs to be replaced with the project ID, and the `:gro The resulting cURL call for a project with ID `5` and a group ID of `17` is then: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/share/17 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/share/17" ``` NOTE: **Note:** @@ -521,10 +525,10 @@ https://gitlab.example.com/api/v4/projects/import `variables` is a parameter of type `array` containing hash key/value pairs `[{ 'key' => 'UPLOAD_TO_S3', 'value' => 'true' }]`: ```shell -curl --globoff --request POST --header "PRIVATE-TOKEN: ********************" \ +curl --globoff --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ "https://gitlab.example.com/api/v4/projects/169/pipeline?ref=master&variables[][key]=VAR1&variables[][value]=hello&variables[][key]=VAR2&variables[][value]=world" -curl --request POST --header "PRIVATE-TOKEN: ********************" \ +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ --header "Content-Type: application/json" \ --data '{ "ref": "master", "variables": [ {"key": "VAR1", "value": "hello"}, {"key": "VAR2", "value": "world"} ] }' \ "https://gitlab.example.com/api/v4/projects/169/pipeline" diff --git a/doc/api/access_requests.md b/doc/api/access_requests.md index fd667073680..53198d05b46 100644 --- a/doc/api/access_requests.md +++ b/doc/api/access_requests.md @@ -6,14 +6,12 @@ The access levels are defined in the `Gitlab::Access` module. Currently, these levels are recognized: -```plaintext -0 => No access -10 => Guest access -20 => Reporter access -30 => Developer access -40 => Maintainer access -50 => Owner access # Only valid for groups -``` +- No access (`0`) +- Guest (`10`) +- Reporter (`20`) +- Developer (`30`) +- Maintainer (`40`) +- Owner (`50`) - Only valid to set for groups ## List access requests for a group or project @@ -31,8 +29,8 @@ GET /projects/:id/access_requests Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/access_requests -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/access_requests +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/access_requests" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/access_requests" ``` Example response: @@ -74,8 +72,8 @@ POST /projects/:id/access_requests Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/access_requests -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/access_requests +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/access_requests" +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/access_requests" ``` Example response: @@ -109,8 +107,8 @@ PUT /projects/:id/access_requests/:user_id/approve Example request: ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/access_requests/:user_id/approve?access_level=20 -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/access_requests/:user_id/approve?access_level=20 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/access_requests/:user_id/approve?access_level=20" +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/access_requests/:user_id/approve?access_level=20" ``` Example response: @@ -143,6 +141,6 @@ DELETE /projects/:id/access_requests/:user_id Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/access_requests/:user_id -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/access_requests/:user_id +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/access_requests/:user_id" +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/access_requests/:user_id" ``` diff --git a/doc/api/admin_sidekiq_queues.md b/doc/api/admin_sidekiq_queues.md index 9d053714b54..32d336e79fe 100644 --- a/doc/api/admin_sidekiq_queues.md +++ b/doc/api/admin_sidekiq_queues.md @@ -33,7 +33,7 @@ DELETE /admin/sidekiq/queues/:queue_name At least one attribute, other than `queue_name`, is required. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/admin/sidekiq/queues/authorized_projects?user=root +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/admin/sidekiq/queues/authorized_projects?user=root" ``` Example response: diff --git a/doc/api/appearance.md b/doc/api/appearance.md index 733d71ee222..47a9d48a4ae 100644 --- a/doc/api/appearance.md +++ b/doc/api/appearance.md @@ -1,6 +1,6 @@ # Appearance API **(CORE ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/16647) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16647) in GitLab 12.7. Appearance API allows you to maintain GitLab's appearance as if using the GitLab UI at `/admin/appearance`. The API requires administrator privileges. @@ -14,7 +14,7 @@ GET /application/appearance ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/appearance +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/appearance" ``` Example response: @@ -50,7 +50,7 @@ PUT /application/appearance | `description` | string | no | Markdown text shown on the sign in / sign up page | `logo` | mixed | no | Instance image used on the sign in / sign up page | `header_logo` | mixed | no | Instance image used for the main navigation bar -| `favicon` | mixed | no | Instance favicon in .ico/.png format +| `favicon` | mixed | no | Instance favicon in `.ico` or `.png` format | `new_project_guidelines` | string | no | Markdown text shown on the new project page | `profile_image_guidelines` | string | no | Markdown text shown on the profile page below Public Avatar | `header_message` | string | no | Message within the system header bar @@ -60,7 +60,7 @@ PUT /application/appearance | `email_header_and_footer_enabled` | boolean | no | Add header and footer to all outgoing emails if enabled ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/appearance?email_header_and_footer_enabled=true&header_message=test +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/appearance?email_header_and_footer_enabled=true&header_message=test" ``` Example response: diff --git a/doc/api/applications.md b/doc/api/applications.md index 5d4a8b3a99f..379f346c019 100644 --- a/doc/api/applications.md +++ b/doc/api/applications.md @@ -32,7 +32,7 @@ Parameters: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "name=MyApplication&redirect_uri=http://redirect.uri&scopes=" https://gitlab.example.com/api/v4/applications +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "name=MyApplication&redirect_uri=http://redirect.uri&scopes=" "https://gitlab.example.com/api/v4/applications" ``` Example response: @@ -59,7 +59,7 @@ GET /applications Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/applications +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/applications" ``` Example response: @@ -98,5 +98,5 @@ Parameters: Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/applications/:id +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/applications/:id" ``` diff --git a/doc/api/audit_events.md b/doc/api/audit_events.md index 7754f431110..ce2a9afd53c 100644 --- a/doc/api/audit_events.md +++ b/doc/api/audit_events.md @@ -25,7 +25,7 @@ are paginated. Read more on [pagination](README.md#pagination). ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/audit_events +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/audit_events" ``` Example response: @@ -96,7 +96,7 @@ GET /audit_events/:id | `id` | integer | yes | The ID of the audit event | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/audit_events/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/audit_events/1" ``` Example response: @@ -122,7 +122,7 @@ Example response: ## Group Audit Events **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34078) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34078) in GitLab 12.5. The Group Audit Events API allows you to retrieve [group audit events](../administration/audit_events.md#group-events-starter). @@ -146,7 +146,7 @@ are paginated. Read more on [pagination](README.md#pagination). ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/groups/60/audit_events +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/groups/60/audit_events" ``` Example response: @@ -202,7 +202,7 @@ GET /groups/:id/audit_events/:audit_event_id | `audit_event_id` | integer | yes | The ID of the audit event | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/groups/60/audit_events/2 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/groups/60/audit_events/2" ``` Example response: @@ -225,3 +225,115 @@ Example response: "created_at": "2019-08-28T19:36:44.162Z" } ``` + +## Project Audit Events **(STARTER)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219238) in GitLab 13.1. + +The Project Audit Events API allows you to retrieve [project audit events](../administration/audit_events.md#project-events-starter). + +To retrieve project audit events using the API, you must [authenticate yourself](README.md#authentication) as a Maintainer or an Owner of the project. + +### Retrieve all project audit events + +```plaintext +GET /projects/:id/audit_events +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `created_after` | string | no | Return project audit events created on or after the given time. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ | +| `created_before` | string | no | Return project audit events created on or before the given time. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ | + +By default, `GET` requests return 20 results at a time because the API results +are paginated. + +Read more on [pagination](README.md#pagination). + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/projects/7/audit_events +``` + +Example response: + +```json +[ + { + "id": 5, + "author_id": 1, + "entity_id": 7, + "entity_type": "Project", + "details": { + "change": "prevent merge request approval from reviewers", + "from": "", + "to": "true", + "author_name": "Administrator", + "target_id": 7, + "target_type": "Project", + "target_details": "twitter/typeahead-js", + "ip_address": "127.0.0.1", + "entity_path": "twitter/typeahead-js" + }, + "created_at": "2020-05-26T22:55:04.230Z" + }, + { + "id": 4, + "author_id": 1, + "entity_id": 7, + "entity_type": "Project", + "details": { + "change": "prevent merge request approval from authors", + "from": "false", + "to": "true", + "author_name": "Administrator", + "target_id": 7, + "target_type": "Project", + "target_details": "twitter/typeahead-js", + "ip_address": "127.0.0.1", + "entity_path": "twitter/typeahead-js" + }, + "created_at": "2020-05-26T22:55:04.218Z" + } +] +``` + +### Retrieve a specific project audit event + +Only available to project maintainers or owners. + +```plaintext +GET /projects/:id/audit_events/:audit_event_id +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `audit_event_id` | integer | yes | The ID of the audit event | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/projects/7/audit_events/5 +``` + +Example response: + +```json +{ + "id": 5, + "author_id": 1, + "entity_id": 7, + "entity_type": "Project", + "details": { + "change": "prevent merge request approval from reviewers", + "from": "", + "to": "true", + "author_name": "Administrator", + "target_id": 7, + "target_type": "Project", + "target_details": "twitter/typeahead-js", + "ip_address": "127.0.0.1", + "entity_path": "twitter/typeahead-js" + }, + "created_at": "2020-05-26T22:55:04.230Z" +} +``` diff --git a/doc/api/avatar.md b/doc/api/avatar.md index 308c0de25f4..223704d3e6c 100644 --- a/doc/api/avatar.md +++ b/doc/api/avatar.md @@ -29,7 +29,7 @@ Parameters: Example request: ```shell -curl https://gitlab.example.com/api/v4/avatar?email=admin@example.com&size=32 +curl "https://gitlab.example.com/api/v4/avatar?email=admin@example.com&size=32" ``` Example response: diff --git a/doc/api/award_emoji.md b/doc/api/award_emoji.md index 37b3cd32f89..6e8739df13c 100644 --- a/doc/api/award_emoji.md +++ b/doc/api/award_emoji.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Award Emoji API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4575) in GitLab 8.9. Snippet support added in 8.12. @@ -36,7 +42,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji" ``` Example response: @@ -99,7 +105,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji/1" ``` Example response: @@ -142,7 +148,7 @@ Parameters: | `name` | string | yes | Name of the emoji without colons. | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji?name=blowfish +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji?name=blowfish" ``` Example Response: @@ -188,7 +194,7 @@ Parameters: | `award_id` | integer | yes | ID of an award emoji. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji/344 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji/344" ``` ## Award Emoji on Comments @@ -219,7 +225,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/notes/1/award_emoji +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/notes/1/award_emoji" ``` Example response: @@ -265,7 +271,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/notes/1/award_emoji/2 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/notes/1/award_emoji/2" ``` Example response: @@ -309,7 +315,7 @@ Parameters: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/notes/1/award_emoji?name=rocket +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/notes/1/award_emoji?name=rocket" ``` Example response: @@ -356,5 +362,5 @@ Parameters: Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji/345 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji/345" ``` diff --git a/doc/api/boards.md b/doc/api/boards.md index 8af23527931..155a876e76a 100644 --- a/doc/api/boards.md +++ b/doc/api/boards.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Issue Boards API Every API call to boards must be authenticated. @@ -18,7 +24,7 @@ GET /projects/:id/boards | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards" ``` Example response: @@ -50,7 +56,8 @@ Example response: }, "position" : 1, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 2, @@ -61,7 +68,8 @@ Example response: }, "position" : 2, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 3, @@ -72,7 +80,8 @@ Example response: }, "position" : 3, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null } ] } @@ -93,7 +102,7 @@ GET /projects/:id/boards/:board_id | `board_id` | integer | yes | The ID of a board | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1" ``` Example response: @@ -125,7 +134,8 @@ Example response: }, "position" : 1, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 2, @@ -136,7 +146,8 @@ Example response: }, "position" : 2, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 3, @@ -147,7 +158,8 @@ Example response: }, "position" : 3, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null } ] } @@ -167,7 +179,7 @@ POST /projects/:id/boards | `name` | string | yes | The name of the new board | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards?name=newboard +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards?name=newboard" ``` Example response: @@ -199,7 +211,8 @@ Example response: }, "position" : 1, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 2, @@ -210,7 +223,8 @@ Example response: }, "position" : 2, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 3, @@ -221,7 +235,8 @@ Example response: }, "position" : 3, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null } ] } @@ -248,7 +263,7 @@ PUT /projects/:id/boards/:board_id | `weight` | integer | no | The weight range from 0 to 9, to which the board should be scoped to | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1?name=new_name&milestone_id=43&assignee_id=1&labels=Doing&weight=4 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1?name=new_name&milestone_id=43&assignee_id=1&labels=Doing&weight=4" ``` Example response: @@ -322,7 +337,7 @@ DELETE /projects/:id/boards/:board_id | `board_id` | integer | yes | The ID of a board | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1" ``` ## List board lists @@ -340,7 +355,7 @@ GET /projects/:id/boards/:board_id/lists | `board_id` | integer | yes | The ID of a board | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1/lists +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1/lists" ``` Example response: @@ -356,7 +371,8 @@ Example response: }, "position" : 1, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 2, @@ -367,7 +383,8 @@ Example response: }, "position" : 2, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null }, { "id" : 3, @@ -378,7 +395,8 @@ Example response: }, "position" : 3, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null } ] ``` @@ -398,7 +416,7 @@ GET /projects/:id/boards/:board_id/lists/:list_id | `list_id`| integer | yes | The ID of a board's list | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1/lists/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1/lists/1" ``` Example response: @@ -413,7 +431,8 @@ Example response: }, "position" : 1, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null } ``` @@ -440,7 +459,7 @@ Check the [Issue Board docs](../user/project/issue_board.md#summary-of-features- for more information regarding the required license for each list type. ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1/lists?label_id=5 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1/lists?label_id=5" ``` Example response: @@ -455,7 +474,8 @@ Example response: }, "position" : 1, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null } ``` @@ -475,7 +495,7 @@ PUT /projects/:id/boards/:board_id/lists/:list_id | `position` | integer | yes | The position of the list | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1/lists/1?position=2 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1/lists/1?position=2" ``` Example response: @@ -490,7 +510,8 @@ Example response: }, "position" : 1, "max_issue_count": 0, - "max_issue_weight": 0 + "max_issue_weight": 0, + "limit_metric": null } ``` @@ -509,5 +530,5 @@ DELETE /projects/:id/boards/:board_id/lists/:list_id | `list_id` | integer | yes | The ID of a board's list | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/boards/1/lists/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/boards/1/lists/1" ``` diff --git a/doc/api/branches.md b/doc/api/branches.md index 7d14a3d54b9..7a64f62189e 100644 --- a/doc/api/branches.md +++ b/doc/api/branches.md @@ -26,7 +26,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/repository/branches +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/branches" ``` Example response: @@ -83,7 +83,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/repository/branches/master +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/branches/master" ``` Example response: @@ -145,7 +145,7 @@ Parameters: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/repository/branches?branch=newbranch&ref=master +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/branches?branch=newbranch&ref=master" ``` Example response: @@ -199,7 +199,7 @@ Parameters: Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/repository/branches/newbranch +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/branches/newbranch" ``` ## Delete merged branches @@ -222,5 +222,5 @@ Parameters: Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/repository/merged_branches +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/merged_branches" ``` diff --git a/doc/api/broadcast_messages.md b/doc/api/broadcast_messages.md index db2198dc162..37156186d03 100644 --- a/doc/api/broadcast_messages.md +++ b/doc/api/broadcast_messages.md @@ -20,7 +20,7 @@ GET /broadcast_messages Example request: ```shell -curl https://gitlab.example.com/api/v4/broadcast_messages +curl "https://gitlab.example.com/api/v4/broadcast_messages" ``` Example response: @@ -59,7 +59,7 @@ Parameters: Example request: ```shell -curl https://gitlab.example.com/api/v4/broadcast_messages/1 +curl "https://gitlab.example.com/api/v4/broadcast_messages/1" ``` Example response: @@ -103,7 +103,7 @@ Parameters: Example request: ```shell -curl --data "message=Deploy in progress&color=#cecece" --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/broadcast_messages +curl --data "message=Deploy in progress&color=#cecece" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/broadcast_messages" ``` Example response: @@ -148,7 +148,7 @@ Parameters: Example request: ```shell -curl --request PUT --data "message=Update message&color=#000" --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/broadcast_messages/1 +curl --request PUT --data "message=Update message&color=#000" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/broadcast_messages/1" ``` Example response: @@ -185,5 +185,5 @@ Parameters: Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/broadcast_messages/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/broadcast_messages/1" ``` diff --git a/doc/api/commits.md b/doc/api/commits.md index 98a8e4ea2ce..9be4ce4fcdb 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -132,7 +132,7 @@ PAYLOAD=$(cat << 'JSON' } JSON ) -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data "$PAYLOAD" https://gitlab.example.com/api/v4/projects/1/repository/commits +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data "$PAYLOAD" "https://gitlab.example.com/api/v4/projects/1/repository/commits" ``` Example response: @@ -206,7 +206,7 @@ Parameters: | `stats` | boolean | no | Include commit stats. Default is true | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/commits/master +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/commits/master" ``` Example response: @@ -498,7 +498,7 @@ POST /projects/:id/repository/commits/:sha/comments | `line_type` | string | no | The line type. Takes `new` or `old` as arguments | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "note=Nice picture man\!" --form "path=dudeism.md" --form "line=11" --form "line_type=new" https://gitlab.example.com/api/v4/projects/17/repository/commits/18f3e63d05582537db6d183d9d557be09e1f90c8/comments +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "note=Nice picture man\!" --form "path=dudeism.md" --form "line=11" --form "line_type=new" "https://gitlab.example.com/api/v4/projects/17/repository/commits/18f3e63d05582537db6d183d9d557be09e1f90c8/comments" ``` Example response: diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index 9ec4373c92c..d4a4fc1a733 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -1,6 +1,6 @@ # Container Registry API -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/55978) in GitLab 11.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55978) in GitLab 11.8. This is the API docs of the [GitLab Container Registry](../user/packages/container_registry/index.md). @@ -19,6 +19,7 @@ GET /projects/:id/registry/repositories | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) accessible by the authenticated user. | | `tags` | boolean | no | If the parameter is included as true, each repository will include an array of `"tags"` in the response. | | `name` | string | no | Returns a list of repositories with a name that matches the value. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29763) in GitLab 13.0). | +| `tags_count` | boolean | no | If the parameter is included as true, each repository will include `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories" @@ -60,9 +61,10 @@ GET /groups/:id/registry/repositories | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) accessible by the authenticated user. | | `tags` | boolean | no | If the parameter is included as true, each repository will include an array of `"tags"` in the response. | | `name` | string | no | Returns a list of repositories with a name that matches the value. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29763) in GitLab 13.0). | +| `tags_count` | boolean | no | If the parameter is included as true, each repository will include `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/2/registry/repositories?tags=1" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/2/registry/repositories?tags=1&tags_count=true" ``` Example response: @@ -76,6 +78,7 @@ Example response: "project_id": 9, "location": "gitlab.example.com:5000/group/project", "created_at": "2019-01-10T13:38:57.391Z", + "tags_count": 1, "tags": [ { "name": "0.0.1", @@ -91,6 +94,7 @@ Example response: "project_id": 11, "location": "gitlab.example.com:5000/group/other_project", "created_at": "2019-01-10T13:39:08.229Z", + "tags_count": 3, "tags": [ { "name": "0.0.1", @@ -236,9 +240,9 @@ DELETE /projects/:id/registry/repositories/:repository_id/tags | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | | `repository_id` | integer | yes | The ID of registry repository. | -| `name_regex` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`. **Note:** `name_regex` is deprecated in favor of `name_regex_delete`.| -| `name_regex_delete` | string | yes | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`.| -| `name_regex_keep` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to keep. This value will override any matches from `name_regex_delete`. Note: setting to `.*` will result in a no-op. | +| `name_regex` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`. **Note:** `name_regex` is deprecated in favor of `name_regex_delete`. This field is validated. | +| `name_regex_delete` | string | yes | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to delete. To delete all tags specify `.*`. This field is validated. | +| `name_regex_keep` | string | no | The [re2](https://github.com/google/re2/wiki/Syntax) regex of the name to keep. This value will override any matches from `name_regex_delete`. This field is validated. Note: setting to `.*` will result in a no-op. | | `keep_n` | integer | no | The amount of latest tags of given name to keep. | | `older_than` | string | no | Tags to delete that are older than the given time, written in human readable form `1h`, `1d`, `1month`. | @@ -260,7 +264,7 @@ This action does not delete blobs. In order to delete them and recycle disk spac NOTE: **Note:** Since GitLab 12.4, individual tags are deleted. -For more details, see the [discussion](https://gitlab.com/gitlab-org/gitlab/issues/15737). +For more details, see the [discussion](https://gitlab.com/gitlab-org/gitlab/-/issues/15737). Examples: diff --git a/doc/api/custom_attributes.md b/doc/api/custom_attributes.md index 20b364993ae..07ece99f9b1 100644 --- a/doc/api/custom_attributes.md +++ b/doc/api/custom_attributes.md @@ -20,7 +20,7 @@ GET /projects/:id/custom_attributes | `id` | integer | yes | The ID of a resource | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/42/custom_attributes +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/42/custom_attributes" ``` Example response: @@ -54,7 +54,7 @@ GET /projects/:id/custom_attributes/:key | `key` | string | yes | The key of the custom attribute | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/42/custom_attributes/location +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/42/custom_attributes/location" ``` Example response: @@ -84,7 +84,7 @@ PUT /projects/:id/custom_attributes/:key | `value` | string | yes | The value of the custom attribute | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "value=Greenland" https://gitlab.example.com/api/v4/users/42/custom_attributes/location +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "value=Greenland" "https://gitlab.example.com/api/v4/users/42/custom_attributes/location" ``` Example response: @@ -112,5 +112,5 @@ DELETE /projects/:id/custom_attributes/:key | `key` | string | yes | The key of the custom attribute | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/42/custom_attributes/location +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/42/custom_attributes/location" ``` diff --git a/doc/api/dependencies.md b/doc/api/dependencies.md index bb7e5ae238d..56d33bf151a 100644 --- a/doc/api/dependencies.md +++ b/doc/api/dependencies.md @@ -28,7 +28,7 @@ GET /projects/:id/dependencies?package_manager=yarn,bundler | `package_manager` | string array | no | Returns dependencies belonging to specified package manager. Valid values: `bundler`, `composer`, `maven`, `npm`, `pip` or `yarn`. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/dependencies +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/dependencies" ``` Example response: diff --git a/doc/api/deploy_keys.md b/doc/api/deploy_keys.md index a7acc0c2b55..1634d07768a 100644 --- a/doc/api/deploy_keys.md +++ b/doc/api/deploy_keys.md @@ -185,7 +185,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git Enables a deploy key for a project so this can be used. Returns the enabled key, with a status code 201 when successful. ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/deploy_keys/13/enable +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/deploy_keys/13/enable" ``` | Attribute | Type | Required | Description | @@ -213,19 +213,19 @@ First, find the ID of the projects you're interested in, by either listing all projects: ```shell -curl --header 'PRIVATE-TOKEN: <your_access_token>' https://gitlab.example.com/api/v4/projects +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects" ``` Or finding the ID of a group: ```shell -curl --header 'PRIVATE-TOKEN: <your_access_token>' https://gitlab.example.com/api/v4/groups +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups" ``` Then listing all projects in that group (for example, group 1234): ```shell -curl --header 'PRIVATE-TOKEN: <your_access_token>' https://gitlab.example.com/api/v4/groups/1234 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1234" ``` With those IDs, add the same deploy key to all: @@ -233,6 +233,6 @@ With those IDs, add the same deploy key to all: ```shell for project_id in 321 456 987; do curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" \ - --data '{"title": "my key", "key": "ssh-rsa AAAA..."}' https://gitlab.example.com/api/v4/projects/${project_id}/deploy_keys + --data '{"title": "my key", "key": "ssh-rsa AAAA..."}' "https://gitlab.example.com/api/v4/projects/${project_id}/deploy_keys" done ``` diff --git a/doc/api/deploy_tokens.md b/doc/api/deploy_tokens.md index 6e732a43da0..bfef61dbda8 100644 --- a/doc/api/deploy_tokens.md +++ b/doc/api/deploy_tokens.md @@ -2,7 +2,7 @@ ## List all deploy tokens -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21811) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21811) in GitLab 12.9. Get a list of all deploy tokens across the GitLab instance. This endpoint requires admin access. @@ -39,7 +39,7 @@ Project deploy token API endpoints require project maintainer access or higher. ### List project deploy tokens -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21811) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21811) in GitLab 12.9. Get a list of a project's deploy tokens. @@ -78,7 +78,7 @@ Example response: ### Create a project deploy token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21811) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21811) in GitLab 12.9. Creates a new deploy token for a project. @@ -115,7 +115,7 @@ Example response: ### Delete a project deploy token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21811) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21811) in GitLab 12.9. Removes a deploy token from the project. @@ -140,7 +140,7 @@ These endpoints require group maintainer access or higher. ### List group deploy tokens -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21811) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21811) in GitLab 12.9. Get a list of a group's deploy tokens @@ -179,7 +179,7 @@ Example response: ### Create a group deploy token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21811) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21811) in GitLab 12.9. Creates a new deploy token for a group. @@ -218,7 +218,7 @@ Example response: ### Delete a group deploy token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21811) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21811) in GitLab 12.9. Removes a deploy token from the group. diff --git a/doc/api/deployments.md b/doc/api/deployments.md index 250c63a8520..8c952ba07b1 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -364,7 +364,7 @@ Example of a response: ## List of merge requests associated with a deployment -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/35739) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35739) in GitLab 12.7. This API retrieves the list of merge requests shipped with a given deployment: diff --git a/doc/api/discussions.md b/doc/api/discussions.md index 572e174201d..1f509a7aadc 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Discussions API Discussions are a set of related notes on: @@ -111,7 +117,7 @@ GET /projects/:id/issues/:issue_iid/discussions ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/discussions +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions" ``` ### Get single issue discussion item @@ -131,7 +137,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a discussion item | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7" ``` ### Create new issue thread @@ -152,7 +158,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/discussions?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions?body=comment" ``` ### Add note to existing issue thread @@ -178,7 +184,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" ``` ### Modify existing issue thread note @@ -200,7 +206,7 @@ Parameters: | `body` | string | yes | The content of the note/reply | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment" ``` ### Delete an issue thread note @@ -221,7 +227,7 @@ Parameters: | `note_id` | integer | yes | The ID of a discussion note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/636 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/636" ``` ## Snippets @@ -319,7 +325,7 @@ GET /projects/:id/snippets/:snippet_id/discussions ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions" ``` ### Get single snippet discussion item @@ -339,7 +345,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a discussion item | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7" ``` ### Create new snippet thread @@ -361,7 +367,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions?body=comment" ``` ### Add note to existing snippet thread @@ -384,7 +390,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" ``` ### Modify existing snippet thread note @@ -406,7 +412,7 @@ Parameters: | `body` | string | yes | The content of the note/reply | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment" ``` ### Delete a snippet thread note @@ -427,7 +433,7 @@ Parameters: | `note_id` | integer | yes | The ID of a discussion note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/636 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/636" ``` ## Epics **(ULTIMATE)** @@ -526,7 +532,7 @@ GET /groups/:id/epics/:epic_id/discussions ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/discussions +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions" ``` ### Get single epic discussion item @@ -546,7 +552,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a discussion item | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7" ``` ### Create new epic thread @@ -568,7 +574,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/discussions?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions?body=comment" ``` ### Add note to existing epic thread @@ -592,7 +598,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" ``` ### Modify existing epic thread note @@ -614,7 +620,7 @@ Parameters: | `body` | string | yes | The content of note/reply | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment" ``` ### Delete an epic thread note @@ -635,7 +641,7 @@ Parameters: | `note_id` | integer | yes | The ID of a thread note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/636 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/636" ``` ## Merge requests @@ -777,7 +783,9 @@ Diff comments also contain position: "new_line": 27, "line_range": { "start_line_code": "588440f66559714280628a4f9799f0c4eb880a4a_10_10", - "end_line_code": "588440f66559714280628a4f9799f0c4eb880a4a_11_11" + "start_line_type": "new", + "end_line_code": "588440f66559714280628a4f9799f0c4eb880a4a_11_11", + "end_line_type": "old" } }, "resolved": false, @@ -790,7 +798,7 @@ Diff comments also contain position: ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions" ``` ### Get single merge request discussion item @@ -810,7 +818,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a discussion item | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7 ``` ### Create new merge request thread @@ -842,13 +850,15 @@ Parameters: | `position[line_range]` | hash | no | Line range for a multi-line diff note | | `position[line_range][start_line_code]` | string | yes | Line code for the start line | | `position[line_range][end_line_code]` | string | yes | Line code for the end line | +| `position[line_range][start_line_type]` | string | yes | Line type for the start line | +| `position[line_range][end_line_type]` | string | yes | Line type for the end line | | `position[width]` | integer | no | Width of the image (for 'image' diff notes) | | `position[height]` | integer | no | Height of the image (for 'image' diff notes) | | `position[x]` | integer | no | X coordinate (for 'image' diff notes) | | `position[y]` | integer | no | Y coordinate (for 'image' diff notes) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions?body=comment" ``` ### Resolve a merge request thread @@ -869,7 +879,7 @@ Parameters: | `resolved` | boolean | yes | Resolve/unresolve the discussion | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7?resolved=true +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7?resolved=true" ``` ### Add note to existing merge request thread @@ -893,7 +903,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" ``` ### Modify an existing merge request thread note @@ -916,13 +926,13 @@ Parameters: | `resolved` | boolean | no | Resolve/unresolve the note (exactly one of `body` or `resolved` must be set | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment" ``` Resolving a note: ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?resolved=true +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?resolved=true" ``` ### Delete a merge request thread note @@ -943,7 +953,7 @@ Parameters: | `note_id` | integer | yes | The ID of a thread note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/636 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/636" ``` ## Commits @@ -1086,7 +1096,7 @@ Diff comments contain also position: ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/commits/11/discussions +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions" ``` ### Get single commit discussion item @@ -1106,7 +1116,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a discussion item | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7" ``` ### Create new commit thread @@ -1141,7 +1151,7 @@ Parameters: | `position[y]` | integer | no | Y coordinate (for 'image' diff notes) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/commits/11/discussions?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions?body=comment" ``` ### Add note to existing commit thread @@ -1164,7 +1174,7 @@ Parameters: | `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment ``` ### Modify an existing commit thread note @@ -1186,13 +1196,13 @@ Parameters: | `body` | string | no | The content of a note | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?body=comment" ``` Resolving a note: ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?resolved=true +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes/1108?resolved=true" ``` ### Delete a commit thread note @@ -1213,5 +1223,5 @@ Parameters: | `note_id` | integer | yes | The ID of a thread note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/636 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/636" ``` diff --git a/doc/api/environments.md b/doc/api/environments.md index ffaff5f4f1e..5f6bdc251ba 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -15,7 +15,7 @@ GET /projects/:id/environments | `search` | string | no | Return list of environments matching the search criteria. Mutually exclusive with `name` | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/environments?name=review%2Ffix-foo +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments?name=review%2Ffix-foo" ``` Example response: @@ -141,7 +141,7 @@ Example of response ## Create a new environment -Creates a new environment with the given name and external_url. +Creates a new environment with the given name and `external_url`. It returns `201` if the environment was successfully created, `400` for wrong parameters. @@ -173,7 +173,7 @@ Example response: ## Edit an existing environment -Updates an existing environment's name and/or external_url. +Updates an existing environment's name and/or `external_url`. It returns `200` if the environment was successfully updated. In case of an error, a status code `400` is returned. @@ -186,7 +186,7 @@ PUT /projects/:id/environments/:environments_id | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `environment_id` | integer | yes | The ID of the environment | | `name` | string | no | The new name of the environment | -| `external_url` | string | no | The new external_url | +| `external_url` | string | no | The new `external_url` | ```shell curl --request PUT --data "name=staging&external_url=https://staging.example.gitlab.com" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments/1" diff --git a/doc/api/epic_issues.md b/doc/api/epic_issues.md index b001749ff5b..86f88b0322f 100644 --- a/doc/api/epic_issues.md +++ b/doc/api/epic_issues.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Portfolio Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Epic Issues API **(ULTIMATE)** Every API call to epic_issues must be authenticated. @@ -20,7 +26,7 @@ GET /groups/:id/epics/:epic_iid/issues | `epic_iid` | integer/string | yes | The internal ID of the epic. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/issues/ +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/issues/" ``` Example response: @@ -117,7 +123,7 @@ POST /groups/:id/epics/:epic_iid/issues/:issue_id | `issue_id` | integer/string | yes | The ID of the issue. | ```shell -curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/issues/55 +curl --header POST "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/issues/55" ``` Example response: @@ -223,7 +229,7 @@ DELETE /groups/:id/epics/:epic_iid/issues/:epic_issue_id | `epic_issue_id` | integer/string | yes | The ID of the issue - epic association. | ```shell -curl --header DELETE "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/issues/11 +curl --header DELETE "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/issues/11" ``` Example response: @@ -331,7 +337,7 @@ PUT /groups/:id/epics/:epic_iid/issues/:epic_issue_id | `move_after_id` | integer/string | no | The ID of the issue - epic association that should be placed after the link in the question. | ```shell -curl --header PUT "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/issues/11?move_before_id=20 +curl --header PUT "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/issues/11?move_before_id=20" ``` Example response: diff --git a/doc/api/epic_links.md b/doc/api/epic_links.md index 4f45fbde9e7..756a41a0680 100644 --- a/doc/api/epic_links.md +++ b/doc/api/epic_links.md @@ -25,7 +25,7 @@ GET /groups/:id/epics/:epic_iid/epics | `epic_iid` | integer | yes | The internal ID of the epic. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/epics/ +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/epics/" ``` Example response: @@ -80,7 +80,7 @@ POST /groups/:id/epics/:epic_iid/epics | `child_epic_id` | integer | yes | The global ID of the child epic. Internal ID can't be used because they can conflict with epics from other groups. | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/epics/6 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/epics/6" ``` Example response: @@ -133,7 +133,7 @@ POST /groups/:id/epics/:epic_iid/epics | `title` | string | yes | The title of a newly created epic. | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/epics?title=Newpic +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/epics?title=Newpic" ``` Example response: @@ -168,7 +168,7 @@ PUT /groups/:id/epics/:epic_iid/epics/:child_epic_id | `move_after_id` | integer | no | The global ID of a sibling epic that should be placed after the child epic. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5" ``` Example response: @@ -223,7 +223,7 @@ DELETE /groups/:id/epics/:epic_iid/epics/:child_epic_id | `child_epic_id` | integer | yes | The global ID of the child epic. Internal ID can't be used because they can conflict with epics from other groups. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/4/epics/5" ``` Example response: diff --git a/doc/api/epics.md b/doc/api/epics.md index 6ca6f04b741..a420ef4cd15 100644 --- a/doc/api/epics.md +++ b/doc/api/epics.md @@ -1,7 +1,13 @@ +--- +stage: Plan +group: Portfolio Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Epics API **(PREMIUM)** > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.2. -> - Single-level Epics [were moved](https://gitlab.com/gitlab-org/gitlab/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. +> - Single-level Epics [were moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. Every API call to epic must be authenticated. @@ -70,7 +76,7 @@ GET /groups/:id/epics?state=opened | `my_reaction_emoji` | string | no | Return epics reacted by the authenticated user by the given emoji. `None` returns epics not given a reaction. `Any` returns epics given at least one reaction. Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31479)| ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics" ``` Example response: @@ -177,7 +183,7 @@ GET /groups/:id/epics/:epic_iid | `epic_iid` | integer/string | yes | The internal ID of the epic. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5" ``` Example response: @@ -253,7 +259,7 @@ POST /groups/:id/epics | `parent_id` | integer/string | no | The ID of a parent epic (since 11.11) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics?title=Epic&description=Epic%20description +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics?title=Epic&description=Epic%20description" ``` Example response: @@ -330,7 +336,7 @@ PUT /groups/:id/epics/:epic_iid | `state_event` | string | no | State event for an epic. Set `close` to close the epic and `reopen` to reopen it (since 11.4) | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5?title=New%20Title +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5?title=New%20Title" ``` Example response: @@ -393,7 +399,7 @@ DELETE /groups/:id/epics/:epic_iid | `epic_iid` | integer/string | yes | The internal ID of the epic. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5" ``` ## Create a todo @@ -412,7 +418,7 @@ POST /groups/:id/epics/:epic_iid/todo | `epic_iid` | integer | yes | The internal ID of a group's epic | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/5/todo +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/5/todo" ``` Example response: diff --git a/doc/api/error_tracking.md b/doc/api/error_tracking.md index f44266aa552..e18fbaf25c3 100644 --- a/doc/api/error_tracking.md +++ b/doc/api/error_tracking.md @@ -1,6 +1,6 @@ # Error Tracking settings API -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34940) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34940) in GitLab 12.7. ## Error Tracking project settings @@ -17,7 +17,7 @@ GET /projects/:id/error_tracking/settings | `id` | integer | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/error_tracking/settings +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/error_tracking/settings" ``` Example response: @@ -45,7 +45,7 @@ PATCH /projects/:id/error_tracking/settings | `active` | boolean | yes | Pass `true` to enable the already configured error tracking settings or `false` to disable it. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/error_tracking/settings?active=true +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/error_tracking/settings?active=true" ``` Example response: diff --git a/doc/api/events.md b/doc/api/events.md index e6cb56f1339..99bb6d5af2b 100644 --- a/doc/api/events.md +++ b/doc/api/events.md @@ -72,7 +72,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01&scope=all +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01&scope=all" ``` Example response: @@ -144,7 +144,7 @@ Parameters: | `sort` | string | no | Sort events in `asc` or `desc` order by `created_at`. Default is `desc` | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/:id/events +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/:id/events" ``` Example response: @@ -277,7 +277,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:project_id/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:project_id/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01" ``` Example response: diff --git a/doc/api/feature_flag_specs.md b/doc/api/feature_flag_specs.md index fbe99826b27..52a4864fdc5 100644 --- a/doc/api/feature_flag_specs.md +++ b/doc/api/feature_flag_specs.md @@ -1,8 +1,17 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Feature Flag Specs API **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. + +CAUTION: **Deprecation** +This API is deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). -The API for creating, updating, reading and deleting [Feature Flag Specs](../user/project/operations/feature_flags.md#define-environment-specs). +The API for creating, updating, reading and deleting Feature Flag Specs. Automation engineers benefit from this API by being able to modify Feature Flag Specs without accessing user interface. To manage the [Feature Flag](../user/project/operations/feature_flags.md) resources via public API, please refer to the [Feature Flags API](feature_flags.md) document. @@ -26,7 +35,7 @@ GET /projects/:id/feature_flag_scopes | `environment` | string | yes | The [environment](../ci/environments/index.md) name | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flag_scopes?environment=production +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/feature_flag_scopes?environment=production" ``` Example response: @@ -94,7 +103,7 @@ GET /projects/:id/feature_flags/:name/scopes | `name` | string | yes | The name of the feature flag. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes" ``` Example response: @@ -160,7 +169,7 @@ POST /projects/:id/feature_flags/:name/scopes | `strategies` | json | yes | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | ```shell -curl https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes \ +curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes" \ --header "PRIVATE-TOKEN: <your_access_token>" \ --header "Content-type: application/json" \ --data @- << EOF @@ -205,7 +214,7 @@ GET /projects/:id/feature_flags/:name/scopes/:environment_scope | `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/feature_flags/new_live_trace/scopes/production +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/feature_flags/new_live_trace/scopes/production" ``` Example response: @@ -243,7 +252,7 @@ PUT /projects/:id/feature_flags/:name/scopes/:environment_scope | `strategies` | json | yes | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | ```shell -curl https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production \ +curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production" \ --header "PRIVATE-TOKEN: <your_access_token>" \ --header "Content-type: application/json" \ --data @- << EOF @@ -287,5 +296,5 @@ DELETE /projects/:id/feature_flags/:name/scopes/:environment_scope | `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production +curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production" ``` diff --git a/doc/api/feature_flag_user_lists.md b/doc/api/feature_flag_user_lists.md index 04cad8f2d1d..460f3727819 100644 --- a/doc/api/feature_flag_user_lists.md +++ b/doc/api/feature_flag_user_lists.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Feature flag user lists API **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205409) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. @@ -23,7 +29,7 @@ GET /projects/:id/feature_flags_user_lists | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists" ``` Example response: @@ -66,7 +72,7 @@ POST /projects/:id/feature_flags_user_lists | `user_xids` | string | yes | A comma separated list of user IDs. | ```shell -curl https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists \ +curl "https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists" \ --header "PRIVATE-TOKEN: <your_access_token>" \ --header "Content-type: application/json" \ --data @- << EOF @@ -105,7 +111,7 @@ GET /projects/:id/feature_flags_user_lists/:iid | `iid` | integer/string | yes | The internal ID of the project's feature flag user list. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1" ``` Example response: @@ -138,7 +144,7 @@ PUT /projects/:id/feature_flags_user_lists/:iid | `user_xids` | string | no | A comma separated list of user IDs. | ```shell -curl https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1 \ +curl "https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1" \ --header "PRIVATE-TOKEN: <your_access_token>" \ --header "Content-type: application/json" \ --request PUT \ @@ -177,5 +183,5 @@ DELETE /projects/:id/feature_flags_user_lists/:iid | `iid` | integer/string | yes | The internal ID of the project's feature flag user list | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE "https://gitlab.example.com/api/v4/projects/1/feature_flags_user_lists/1" ``` diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index 3e1b0e05298..f3af662c972 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -1,6 +1,15 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Feature Flags API **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. + +NOTE: **Note** +This API is behind a [feature flag](../user/project/operations/feature_flags.md#feature-flag-behavior-change-in-130). If this flag is not enabled in your environment, you can use the [legacy feature flags API](feature_flags_legacy.md). API for accessing resources of [GitLab Feature Flags](../user/project/operations/feature_flags.md). @@ -25,7 +34,7 @@ GET /projects/:id/feature_flags | `scope` | string | no | The condition of feature flags, one of: `enabled`, `disabled`. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/feature_flags" ``` Example response: @@ -35,113 +44,94 @@ Example response: { "name":"merge_train", "description":"This feature is about merge train", + "version": "new_version_flag", "created_at":"2019-11-04T08:13:51.423Z", "updated_at":"2019-11-04T08:13:51.423Z", - "scopes":[ - { - "id":82, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:51.425Z", - "updated_at":"2019-11-04T08:13:51.425Z" - }, - { - "id":83, - "active":true, - "environment_scope":"review/*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:51.427Z", - "updated_at":"2019-11-04T08:13:51.427Z" - }, - { - "id":84, - "active":false, - "environment_scope":"production", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:51.428Z", - "updated_at":"2019-11-04T08:13:51.428Z" - } + "scopes":[], + "strategies": [ + { + "id": 1, + "name": "userWithId", + "parameters": { + "userIds": "user1" + }, + "scopes": [ + { + "id": 1, + "environment_scope": "production" + } + ] + } ] }, { "name":"new_live_trace", "description":"This is a new live trace feature", + "version": "new_version_flag", "created_at":"2019-11-04T08:13:10.507Z", "updated_at":"2019-11-04T08:13:10.507Z", - "scopes":[ - { - "id":79, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.516Z", - "updated_at":"2019-11-04T08:13:10.516Z" - }, - { - "id":80, - "active":true, - "environment_scope":"staging", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.525Z", - "updated_at":"2019-11-04T08:13:10.525Z" - }, - { - "id":81, - "active":false, - "environment_scope":"production", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.527Z", - "updated_at":"2019-11-04T08:13:10.527Z" - } + "scopes":[] + "strategies": [ + { + "id": 2, + "name": "default", + "parameters": {}, + "scopes": [ + { + "id": 2, + "environment_scope": "staging" + } + ] + } ] } ] ``` -## New feature flag +## Get a single feature flag + +Gets a single feature flag. + +```plaintext +GET /projects/:id/feature_flags/:name +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `name` | string | yes | The name of the feature flag. | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature +``` + +Example response: + +```json +{ + "name": "awesome_feature", + "description": null, + "version": "new_version_flag", + "created_at": "2020-05-13T19:56:33.119Z", + "updated_at": "2020-05-13T19:56:33.119Z", + "scopes": [], + "strategies": [ + { + "id": 36, + "name": "default", + "parameters": {}, + "scopes": [ + { + "id": 37, + "environment_scope": "production" + } + ] + } + ] +} +``` + +## Create a feature flag Creates a new feature flag. @@ -152,22 +142,24 @@ POST /projects/:id/feature_flags | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | -| `description` | string | no | The description of the feature flag. | -| `scopes` | JSON | no | The [feature flag specs](../user/project/operations/feature_flags.md#define-environment-specs) of the feature flag. | -| `scopes:environment_scope` | string | no | The [environment spec](../ci/environments/index.md#scoping-environments-with-specs). | -| `scopes:active` | boolean | no | Whether the spec is active. | -| `scopes:strategies` | JSON | no | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | +| `name` | string | yes | The name of the feature flag. | +| `version` | string | yes | The version of the feature flag. Must be `new_version_flag`. Omit or set to `legacy_flag` to create a [Legacy Feature Flag](feature_flags_legacy.md). | +| `description` | string | no | The description of the feature flag. | +| `strategies` | JSON | no | The feature flag [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies). | +| `strategies:name` | JSON | no | The strategy name. | +| `strategies:parameters` | JSON | no | The strategy parameters. | +| `strategies:scopes` | JSON | no | The scopes for the strategy. | +| `strategies:scopes:environment_scope` | string | no | The environment spec for the scope. | ```shell -curl https://gitlab.example.com/api/v4/projects/1/feature_flags \ +curl "https://gitlab.example.com/api/v4/projects/1/feature_flags" \ --header "PRIVATE-TOKEN: <your_access_token>" \ --header "Content-type: application/json" \ --data @- << EOF { - "name": "awesome_feature", - "scopes": [{ "environment_scope": "*", "active": false, "strategies": [{ "name": "default", "parameters": {} }] }, - { "environment_scope": "production", "active": true, "strategies": [{ "name": "userWithId", "parameters": { "userIds": "1,2,3" } }] }] + "name": "awesome_feature", + "version": "new_version_flag", + "strategies": [{ "name": "default", "parameters": {}, "scopes": [{ "environment_scope": "production" }] }] } EOF ``` @@ -176,121 +168,101 @@ Example response: ```json { - "name":"awesome_feature", - "description":null, - "created_at":"2019-11-04T08:32:27.288Z", - "updated_at":"2019-11-04T08:32:27.288Z", - "scopes":[ - { - "id":85, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:32:29.324Z", - "updated_at":"2019-11-04T08:32:29.324Z" - }, - { - "id":86, - "active":true, - "environment_scope":"production", - "strategies":[ - { - "name":"userWithId", - "parameters":{ - "userIds":"1,2,3" - } - } - ], - "created_at":"2019-11-04T08:32:29.328Z", - "updated_at":"2019-11-04T08:32:29.328Z" - } - ] + "name": "awesome_feature", + "description": null, + "version": "new_version_flag", + "created_at": "2020-05-13T19:56:33.119Z", + "updated_at": "2020-05-13T19:56:33.119Z", + "scopes": [], + "strategies": [ + { + "id": 36, + "name": "default", + "parameters": {}, + "scopes": [ + { + "id": 37, + "environment_scope": "production" + } + ] + } + ] } ``` -## Single feature flag +## Update a feature flag -Gets a single feature flag. +Updates a feature flag. ```plaintext -GET /projects/:id/feature_flags/:name +PUT /projects/:id/feature_flags/:name ``` | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | +| `name` | string | yes | The name of the feature flag. | +| `description` | string | no | The description of the feature flag. | +| `strategies` | JSON | no | The feature flag [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies). | +| `strategies:id` | JSON | no | The feature flag strategy id. | +| `strategies:name` | JSON | no | The strategy name. | +| `strategies:parameters` | JSON | no | The strategy parameters. | +| `strategies:scopes` | JSON | no | The scopes for the strategy. | +| `strategies:scopes:id` | JSON | no | The scopes id. | +| `strategies:scopes:environment_scope` | string | no | The environment spec for the scope. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace +curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature" \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + --header "Content-type: application/json" \ + --data @- << EOF +{ + "strategies": [{ "name": "gradualRolloutUserId", "parameters": { "groupId": "default", "percentage": "25" }, "scopes": [{ "environment_scope": "staging" }] }] +} +EOF ``` Example response: ```json { - "name":"new_live_trace", - "description":"This is a new live trace feature", - "created_at":"2019-11-04T08:13:10.507Z", - "updated_at":"2019-11-04T08:13:10.507Z", - "scopes":[ - { - "id":79, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.516Z", - "updated_at":"2019-11-04T08:13:10.516Z" - }, - { - "id":80, - "active":true, - "environment_scope":"staging", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.525Z", - "updated_at":"2019-11-04T08:13:10.525Z" + "name": "awesome_feature", + "description": null, + "version": "new_version_flag", + "created_at": "2020-05-13T20:10:32.891Z", + "updated_at": "2020-05-13T20:10:32.891Z", + "scopes": [], + "strategies": [ + { + "id": 38, + "name": "gradualRolloutUserId", + "parameters": { + "groupId": "default", + "percentage": "25" }, - { - "id":81, - "active":false, - "environment_scope":"production", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.527Z", - "updated_at":"2019-11-04T08:13:10.527Z" - } - ] + "scopes": [ + { + "id": 40, + "environment_scope": "staging" + } + ] + }, + { + "id": 37, + "name": "default", + "parameters": {}, + "scopes": [ + { + "id": 39, + "environment_scope": "production" + } + ] + } + ] } ``` -## Delete feature flag +## Delete a feature flag Deletes a feature flag. @@ -304,5 +276,5 @@ DELETE /projects/:id/feature_flags/:name | `name` | string | yes | The name of the feature flag. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature +curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE "https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature" ``` diff --git a/doc/api/feature_flags_legacy.md b/doc/api/feature_flags_legacy.md new file mode 100644 index 00000000000..30bae9c5eeb --- /dev/null +++ b/doc/api/feature_flags_legacy.md @@ -0,0 +1,317 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Legacy Feature Flags API **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. + +CAUTION: **Deprecation** +This API is deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). Use [this API](feature_flags.md) instead. + +API for accessing resources of [GitLab Feature Flags](../user/project/operations/feature_flags.md). + +Users with Developer or higher [permissions](../user/permissions.md) can access Feature Flag API. + +## Feature Flags pagination + +By default, `GET` requests return 20 results at a time because the API results +are [paginated](README.md#pagination). + +## List feature flags for a project + +Gets all feature flags of the requested project. + +```plaintext +GET /projects/:id/feature_flags +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `scope` | string | no | The condition of feature flags, one of: `enabled`, `disabled`. | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags +``` + +Example response: + +```json +[ + { + "name":"merge_train", + "description":"This feature is about merge train", + "created_at":"2019-11-04T08:13:51.423Z", + "updated_at":"2019-11-04T08:13:51.423Z", + "scopes":[ + { + "id":82, + "active":false, + "environment_scope":"*", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:51.425Z", + "updated_at":"2019-11-04T08:13:51.425Z" + }, + { + "id":83, + "active":true, + "environment_scope":"review/*", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:51.427Z", + "updated_at":"2019-11-04T08:13:51.427Z" + }, + { + "id":84, + "active":false, + "environment_scope":"production", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:51.428Z", + "updated_at":"2019-11-04T08:13:51.428Z" + } + ] + }, + { + "name":"new_live_trace", + "description":"This is a new live trace feature", + "created_at":"2019-11-04T08:13:10.507Z", + "updated_at":"2019-11-04T08:13:10.507Z", + "scopes":[ + { + "id":79, + "active":false, + "environment_scope":"*", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:10.516Z", + "updated_at":"2019-11-04T08:13:10.516Z" + }, + { + "id":80, + "active":true, + "environment_scope":"staging", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:10.525Z", + "updated_at":"2019-11-04T08:13:10.525Z" + }, + { + "id":81, + "active":false, + "environment_scope":"production", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:10.527Z", + "updated_at":"2019-11-04T08:13:10.527Z" + } + ] + } +] +``` + +## New feature flag + +Creates a new feature flag. + +```plaintext +POST /projects/:id/feature_flags +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `name` | string | yes | The name of the feature flag. | +| `description` | string | no | The description of the feature flag. | +| `scopes` | JSON | no | The feature flag specs of the feature flag. | +| `scopes:environment_scope` | string | no | The environment spec. | +| `scopes:active` | boolean | no | Whether the spec is active. | +| `scopes:strategies` | JSON | no | The [strategies](../user/project/operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | + +```shell +curl https://gitlab.example.com/api/v4/projects/1/feature_flags \ + --header "PRIVATE-TOKEN: <your_access_token>" \ + --header "Content-type: application/json" \ + --data @- << EOF +{ + "name": "awesome_feature", + "scopes": [{ "environment_scope": "*", "active": false, "strategies": [{ "name": "default", "parameters": {} }] }, + { "environment_scope": "production", "active": true, "strategies": [{ "name": "userWithId", "parameters": { "userIds": "1,2,3" } }] }] +} +EOF +``` + +Example response: + +```json +{ + "name":"awesome_feature", + "description":null, + "created_at":"2019-11-04T08:32:27.288Z", + "updated_at":"2019-11-04T08:32:27.288Z", + "scopes":[ + { + "id":85, + "active":false, + "environment_scope":"*", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:32:29.324Z", + "updated_at":"2019-11-04T08:32:29.324Z" + }, + { + "id":86, + "active":true, + "environment_scope":"production", + "strategies":[ + { + "name":"userWithId", + "parameters":{ + "userIds":"1,2,3" + } + } + ], + "created_at":"2019-11-04T08:32:29.328Z", + "updated_at":"2019-11-04T08:32:29.328Z" + } + ] +} +``` + +## Single feature flag + +Gets a single feature flag. + +```plaintext +GET /projects/:id/feature_flags/:name +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `name` | string | yes | The name of the feature flag. | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace +``` + +Example response: + +```json +{ + "name":"new_live_trace", + "description":"This is a new live trace feature", + "created_at":"2019-11-04T08:13:10.507Z", + "updated_at":"2019-11-04T08:13:10.507Z", + "scopes":[ + { + "id":79, + "active":false, + "environment_scope":"*", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:10.516Z", + "updated_at":"2019-11-04T08:13:10.516Z" + }, + { + "id":80, + "active":true, + "environment_scope":"staging", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:10.525Z", + "updated_at":"2019-11-04T08:13:10.525Z" + }, + { + "id":81, + "active":false, + "environment_scope":"production", + "strategies":[ + { + "name":"default", + "parameters":{ + + } + } + ], + "created_at":"2019-11-04T08:13:10.527Z", + "updated_at":"2019-11-04T08:13:10.527Z" + } + ] +} +``` + +## Delete feature flag + +Deletes a feature flag. + +```plaintext +DELETE /projects/:id/feature_flags/:name +``` + +| Attribute | Type | Required | Description | +| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `name` | string | yes | The name of the feature flag. | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" --request DELETE https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature +``` diff --git a/doc/api/features.md b/doc/api/features.md index 03f1663b987..bbf86eca490 100644 --- a/doc/api/features.md +++ b/doc/api/features.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Features flags API This API is for managing Flipper-based [feature flags used in development of GitLab](../development/feature_flags/index.md). @@ -16,7 +22,7 @@ GET /features ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/features +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/features" ``` Example response: @@ -80,7 +86,7 @@ Note that you can enable or disable a feature for a `feature_group`, a `user`, a `group`, and a `project` in a single API call. ```shell -curl --data "value=30" --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/features/new_library +curl --data "value=30" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/features/new_library" ``` Example response: diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 820f78853d0..12f785a3e3d 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -12,7 +12,7 @@ POST /geo_nodes ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes \ +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/geo_nodes" \ --request POST \ -d "name=himynameissomething" \ -d "url=https://another-node.example.com/" @@ -73,7 +73,7 @@ GET /geo_nodes ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/geo_nodes" ``` Example response: @@ -140,7 +140,7 @@ GET /geo_nodes/:id ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/geo_nodes/1" ``` Example response: @@ -287,7 +287,7 @@ GET /geo_nodes/status ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/status +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/geo_nodes/status" ``` Example response: @@ -366,7 +366,9 @@ Example response: "revision": "33d33a096a", "package_files_count": 10, "package_files_checksummed_count": 10, - "package_files_checksum_failed_count": 0 + "package_files_checksum_failed_count": 0, + "package_files_synced_count": 10, + "package_files_failed_count": 5 }, { "geo_node_id": 2, @@ -437,7 +439,9 @@ Example response: "revision": "33d33a096a", "package_files_count": 10, "package_files_checksummed_count": 10, - "package_files_checksum_failed_count": 0 + "package_files_checksum_failed_count": 0, + "package_files_synced_count": 10, + "package_files_failed_count": 5 } ] ``` @@ -452,7 +456,7 @@ GET /geo_nodes/:id/status ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/2/status +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/geo_nodes/2/status" ``` Example response: @@ -530,7 +534,7 @@ GET /geo_nodes/current/failures This endpoint uses [Pagination](README.md#pagination). ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/current/failures +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://primary.example.com/api/v4/geo_nodes/current/failures" ``` Example response: diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 044c3500bf4..d653c4e0f47 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -29,11 +29,11 @@ allows clients to request exactly the data they need, making it possible to get all required data in a limited number of requests. The GraphQL data (fields) can be described in the form of types, -allowing clients to use [clientside GraphQL +allowing clients to use [client-side GraphQL libraries](https://graphql.org/code/#graphql-clients) to consume the API and avoid manual parsing. -Since there's no fixed endpoints and datamodel, new abilities can be +Since there's no fixed endpoints and data model, new abilities can be added to the API without creating breaking changes. This allows us to have a versionless API as described in [the GraphQL documentation](https://graphql.org/learn/best-practices/#versioning). @@ -57,8 +57,10 @@ The GraphQL API includes the following queries at the root level: 1. `project` : Project information, with many of its associations such as issues and merge requests. 1. `group` : Basic group information and epics **(ULTIMATE)** are currently supported. +1. `user` : Information about a particular user. 1. `namespace` : Within a namespace it is also possible to fetch `projects`. 1. `currentUser`: Information about the currently logged in user. +1. `users`: Information about a collection of users. 1. `metaData`: Metadata about GitLab and the GraphQL API. 1. `snippets`: Snippets visible to the currently logged in user. diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 91f1413943c..c4bbe7d969d 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -1,4 +1,31 @@ """ +Represents the access level of a relationship between a User and object that it is related to +""" +type AccessLevel { + """ + Integer representation of access level + """ + integerValue: Int + + """ + String representation of access level + """ + stringValue: AccessLevelEnum +} + +""" +Access level to a resource +""" +enum AccessLevelEnum { + DEVELOPER + GUEST + MAINTAINER + NO_ACCESS + OWNER + REPORTER +} + +""" Autogenerated input type of AddAwardEmoji """ input AddAwardEmojiInput { @@ -141,7 +168,32 @@ type AdminSidekiqQueuesDeleteJobsPayload { """ Describes an alert from the project's Alert Management """ -type AlertManagementAlert { +type AlertManagementAlert implements Noteable { + """ + Assignees of the alert + """ + assignees( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): UserConnection + """ Timestamp the alert was created """ @@ -158,6 +210,31 @@ type AlertManagementAlert { details: JSON """ + All discussions on this noteable + """ + discussions( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): DiscussionConnection! + + """ Timestamp the alert ended """ endedAt: Time @@ -188,6 +265,31 @@ type AlertManagementAlert { monitoringTool: String """ + All notes on this noteable + """ + notes( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): NoteConnection! + + """ Service the alert came from """ service: String @@ -270,22 +372,22 @@ enum AlertManagementAlertSort { """ End time by ascending order """ - END_TIME_ASC + ENDED_AT_ASC """ End time by descending order """ - END_TIME_DESC + ENDED_AT_DESC """ Events count by ascending order """ - EVENTS_COUNT_ASC + EVENT_COUNT_ASC """ Events count by descending order """ - EVENTS_COUNT_DESC + EVENT_COUNT_DESC """ Severity by ascending order @@ -300,12 +402,12 @@ enum AlertManagementAlertSort { """ Start time by ascending order """ - START_TIME_ASC + STARTED_AT_ASC """ Start time by descending order """ - START_TIME_DESC + STARTED_AT_DESC """ Status by ascending order @@ -444,6 +546,61 @@ enum AlertManagementStatus { } """ +Autogenerated input type of AlertSetAssignees +""" +input AlertSetAssigneesInput { + """ + The usernames to assign to the alert. Replaces existing assignees by default. + """ + assigneeUsernames: [String!]! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The iid of the alert to mutate + """ + iid: String! + + """ + The operation to perform. Defaults to REPLACE. + """ + operationMode: MutationOperationMode + + """ + The project the alert to mutate is in + """ + projectPath: ID! +} + +""" +Autogenerated return type of AlertSetAssignees +""" +type AlertSetAssigneesPayload { + """ + The alert after mutation + """ + alert: AlertManagementAlert + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue created after mutation + """ + issue: Issue +} + +""" An emoji awarded by a user. """ type AwardEmoji { @@ -586,7 +743,7 @@ type Board { id: ID! """ - Lists of the project board + Lists of the board """ lists( """ @@ -935,6 +1092,266 @@ type Commit { webUrl: String! } +input CommitAction { + """ + The action to perform, create, delete, move, update, chmod + """ + action: CommitActionMode! + + """ + Content of the file + """ + content: String + + """ + Encoding of the file. Default is text + """ + encoding: CommitEncoding + + """ + Enables/disables the execute flag on the file + """ + executeFilemode: Boolean + + """ + Full path to the file + """ + filePath: String! + + """ + Last known file commit ID + """ + lastCommitId: String + + """ + Original full path to the file being moved + """ + previousPath: String +} + +""" +Mode of a commit action +""" +enum CommitActionMode { + """ + Chmod command + """ + CHMOD + + """ + Create command + """ + CREATE + + """ + Delete command + """ + DELETE + + """ + Move command + """ + MOVE + + """ + Update command + """ + UPDATE +} + +""" +Autogenerated input type of CommitCreate +""" +input CommitCreateInput { + """ + Array of action hashes to commit as a batch + """ + actions: [CommitAction!]! + + """ + Name of the branch + """ + branch: String! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Raw commit message + """ + message: String! + + """ + Project full path the branch is associated with + """ + projectPath: ID! +} + +""" +Autogenerated return type of CommitCreate +""" +type CommitCreatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The commit after mutation + """ + commit: Commit + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +enum CommitEncoding { + """ + Base64 encoding + """ + BASE64 + + """ + Text encoding + """ + TEXT +} + +""" +A tag expiration policy designed to keep only the images that matter most +""" +type ContainerExpirationPolicy { + """ + This container expiration policy schedule + """ + cadence: ContainerExpirationPolicyCadenceEnum! + + """ + Timestamp of when the container expiration policy was created + """ + createdAt: Time! + + """ + Indicates whether this container expiration policy is enabled + """ + enabled: Boolean! + + """ + Number of tags to retain + """ + keepN: ContainerExpirationPolicyKeepEnum + + """ + Tags with names matching this regex pattern will expire + """ + nameRegex: String + + """ + Tags with names matching this regex pattern will be preserved + """ + nameRegexKeep: String + + """ + Next time that this container expiration policy will get executed + """ + nextRunAt: Time + + """ + Tags older that this will expire + """ + olderThan: ContainerExpirationPolicyOlderThanEnum + + """ + Timestamp of when the container expiration policy was updated + """ + updatedAt: Time! +} + +enum ContainerExpirationPolicyCadenceEnum { + """ + Every day + """ + EVERY_DAY + + """ + Every month + """ + EVERY_MONTH + + """ + Every three months + """ + EVERY_THREE_MONTHS + + """ + Every two weeks + """ + EVERY_TWO_WEEKS + + """ + Every week + """ + EVERY_WEEK +} + +enum ContainerExpirationPolicyKeepEnum { + """ + 50 tags per image name + """ + FIFTY_TAGS + + """ + 5 tags per image name + """ + FIVE_TAGS + + """ + 100 tags per image name + """ + ONE_HUNDRED_TAGS + + """ + 1 tag per image name + """ + ONE_TAG + + """ + 10 tags per image name + """ + TEN_TAGS + + """ + 25 tags per image name + """ + TWENTY_FIVE_TAGS +} + +enum ContainerExpirationPolicyOlderThanEnum { + """ + 14 days until tags are automatically removed + """ + FOURTEEN_DAYS + + """ + 90 days until tags are automatically removed + """ + NINETY_DAYS + + """ + 7 days until tags are automatically removed + """ + SEVEN_DAYS + + """ + 30 days until tags are automatically removed + """ + THIRTY_DAYS +} + """ Autogenerated input type of CreateAlertIssue """ @@ -1460,6 +1877,43 @@ type CreateSnippetPayload { snippet: Snippet } +enum DastScanTypeEnum { + """ + Passive DAST scan. This scan will not make active attacks against the target site. + """ + PASSIVE +} + +""" +Autogenerated input type of DeleteAnnotation +""" +input DeleteAnnotationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The global id of the annotation to delete + """ + id: ID! +} + +""" +Autogenerated return type of DeleteAnnotation +""" +type DeleteAnnotationPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + """ The response from the AdminSidekiqQueuesDeleteJobs mutation. """ @@ -2492,7 +2946,7 @@ type DiffRefs { startSha: String! } -type Discussion { +type Discussion implements ResolvableInterface { """ Timestamp of the discussion's creation """ @@ -2532,6 +2986,26 @@ type Discussion { ID used to reply to this discussion """ replyId: ID! + + """ + Indicates if the object can be resolved + """ + resolvable: Boolean! + + """ + Indicates if the object is resolved + """ + resolved: Boolean! + + """ + Timestamp of when the object was resolved + """ + resolvedAt: Time + + """ + User who resolved the object + """ + resolvedBy: User } """ @@ -2570,6 +3044,46 @@ type DiscussionEdge { } """ +Autogenerated input type of DiscussionToggleResolve +""" +input DiscussionToggleResolveInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The global id of the discussion + """ + id: ID! + + """ + Will resolve the discussion when true, and unresolve the discussion when false + """ + resolve: Boolean! +} + +""" +Autogenerated return type of DiscussionToggleResolve +""" +type DiscussionToggleResolvePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The discussion after mutation + """ + discussion: Discussion + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + +""" Autogenerated input type of DismissVulnerability """ input DismissVulnerabilityInput { @@ -4328,6 +4842,46 @@ type Group { ): IterationConnection """ + A label available on this group + """ + label( + """ + Title of the label + """ + title: String! + ): Label + + """ + Labels available on this group + """ + labels( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + A search term to find labels with + """ + searchTerm: String + ): LabelConnection + + """ Indicates if Large File Storage (LFS) is enabled for namespace """ lfsEnabled: Boolean @@ -4610,6 +5164,81 @@ type Group { webUrl: String! } +""" +Represents a Group Member +""" +type GroupMember implements MemberInterface { + """ + GitLab::Access level + """ + accessLevel: AccessLevel + + """ + Date and time the membership was created + """ + createdAt: Time + + """ + User that authorized membership + """ + createdBy: User + + """ + Date and time the membership expires + """ + expiresAt: Time + + """ + Group that a User is a member of + """ + group: Group + + """ + Date and time the membership was last updated + """ + updatedAt: Time + + """ + Permissions for the current user on the resource + """ + userPermissions: GroupPermissions! +} + +""" +The connection type for GroupMember. +""" +type GroupMemberConnection { + """ + A list of edges. + """ + edges: [GroupMemberEdge] + + """ + A list of nodes. + """ + nodes: [GroupMember] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type GroupMemberEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: GroupMember +} + type GroupPermissions { """ Indicates the user can perform `read_group` on this resource @@ -5421,6 +6050,16 @@ type JiraImport { createdAt: Time """ + Count of issues that failed to import + """ + failedToImportCount: Int! + + """ + Count of issues that were successfully imported + """ + importedIssuesCount: Int! + + """ Project key for the imported Jira project """ jiraProjectKey: String! @@ -5434,6 +6073,11 @@ type JiraImport { User that started the Jira import """ scheduledBy: User + + """ + Total count of issues that were attempted to import + """ + totalIssueCount: Int! } """ @@ -5516,6 +6160,98 @@ type JiraImportStartPayload { jiraImport: JiraImport } +""" +Autogenerated input type of JiraImportUsers +""" +input JiraImportUsersInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The project to import the Jira users into + """ + projectPath: ID! + + """ + The index of the record the import should started at, default 0 (50 records returned) + """ + startAt: Int +} + +""" +Autogenerated return type of JiraImportUsers +""" +type JiraImportUsersPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + Users returned from Jira, matched by email and name if possible. + """ + jiraUsers: [JiraUser!] +} + +type JiraProject { + """ + Key of the Jira project + """ + key: String! + + """ + Name of the Jira project + """ + name: String + + """ + ID of the Jira project + """ + projectId: Int! +} + +""" +The connection type for JiraProject. +""" +type JiraProjectConnection { + """ + A list of edges. + """ + edges: [JiraProjectEdge] + + """ + A list of nodes. + """ + nodes: [JiraProject] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type JiraProjectEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: JiraProject +} + type JiraService implements Service { """ Indicates if the service is active @@ -5523,11 +6259,63 @@ type JiraService implements Service { active: Boolean """ + List of Jira projects fetched through Jira REST API + """ + projects( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Project name or key + """ + name: String + ): JiraProjectConnection + + """ Class name of the service """ type: String } +type JiraUser { + """ + Id of the matched GitLab user + """ + gitlabId: Int + + """ + Account id of the Jira user + """ + jiraAccountId: String! + + """ + Display name of the Jira user + """ + jiraDisplayName: String! + + """ + Email of the Jira user, returned only for users with public emails + """ + jiraEmail: String +} + type Label { """ Background color of the label @@ -5639,6 +6427,33 @@ type MarkAsSpamSnippetPayload { snippet: Snippet } +interface MemberInterface { + """ + GitLab::Access level + """ + accessLevel: AccessLevel + + """ + Date and time the membership was created + """ + createdAt: Time + + """ + User that authorized membership + """ + createdBy: User + + """ + Date and time the membership expires + """ + expiresAt: Time + + """ + Date and time the membership was last updated + """ + updatedAt: Time +} + type MergeRequest implements Noteable { """ Indicates if members of the target project can push to the fork @@ -5671,6 +6486,11 @@ type MergeRequest implements Noteable { ): UserConnection """ + User who created this merge request + """ + author: User + + """ Timestamp of when the merge request was created """ createdAt: Time! @@ -5821,6 +6641,11 @@ type MergeRequest implements Noteable { mergeableDiscussionsState: Boolean """ + Timestamp of when the merge request was merged, null if not merged + """ + mergedAt: Time + + """ The milestone of the merge request """ milestone: Milestone @@ -5991,6 +6816,11 @@ type MergeRequest implements Noteable { targetBranch: String! """ + Indicates if the target branch of the merge request exists + """ + targetBranchExists: Boolean! + + """ Target project of the merge request """ targetProject: Project! @@ -6077,6 +6907,61 @@ type MergeRequestConnection { } """ +Autogenerated input type of MergeRequestCreate +""" +input MergeRequestCreateInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Description of the merge request (Markdown rendered as HTML for caching) + """ + description: String + + """ + Project full path the merge request is associated with + """ + projectPath: ID! + + """ + Source branch of the merge request + """ + sourceBranch: String! + + """ + Target branch of the merge request + """ + targetBranch: String! + + """ + Title of the merge request + """ + title: String! +} + +""" +Autogenerated return type of MergeRequestCreate +""" +type MergeRequestCreatePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The merge request after mutation + """ + mergeRequest: MergeRequest +} + +""" An edge in a connection. """ type MergeRequestEdge { @@ -6478,6 +7363,11 @@ type MetricsDashboard { Path to a file with the dashboard definition """ path: String + + """ + Dashboard schema validation warnings + """ + schemaValidationWarnings: [String!] } type MetricsDashboardAnnotation { @@ -6562,11 +7452,21 @@ type Milestone { dueDate: Time """ + Indicates if milestone is at group level + """ + groupMilestone: Boolean! + + """ ID of the milestone """ id: ID! """ + Indicates if milestone is at project level + """ + projectMilestone: Boolean! + + """ Timestamp of the milestone start date """ startDate: Time @@ -6577,6 +7477,11 @@ type Milestone { state: MilestoneStateEnum! """ + Indicates if milestone is at subgroup level + """ + subgroupMilestone: Boolean! + + """ Title of the milestone """ title: String! @@ -6651,7 +7556,9 @@ type Mutation { addAwardEmoji(input: AddAwardEmojiInput!): AddAwardEmojiPayload addProjectToSecurityDashboard(input: AddProjectToSecurityDashboardInput!): AddProjectToSecurityDashboardPayload adminSidekiqQueuesDeleteJobs(input: AdminSidekiqQueuesDeleteJobsInput!): AdminSidekiqQueuesDeleteJobsPayload + alertSetAssignees(input: AlertSetAssigneesInput!): AlertSetAssigneesPayload boardListUpdateLimitMetrics(input: BoardListUpdateLimitMetricsInput!): BoardListUpdateLimitMetricsPayload + commitCreate(input: CommitCreateInput!): CommitCreatePayload createAlertIssue(input: CreateAlertIssueInput!): CreateAlertIssuePayload createAnnotation(input: CreateAnnotationInput!): CreateAnnotationPayload createBranch(input: CreateBranchInput!): CreateBranchPayload @@ -6662,10 +7569,16 @@ type Mutation { createNote(input: CreateNoteInput!): CreateNotePayload createRequirement(input: CreateRequirementInput!): CreateRequirementPayload createSnippet(input: CreateSnippetInput!): CreateSnippetPayload + deleteAnnotation(input: DeleteAnnotationInput!): DeleteAnnotationPayload designManagementDelete(input: DesignManagementDeleteInput!): DesignManagementDeletePayload designManagementUpload(input: DesignManagementUploadInput!): DesignManagementUploadPayload destroyNote(input: DestroyNoteInput!): DestroyNotePayload destroySnippet(input: DestroySnippetInput!): DestroySnippetPayload + + """ + Toggles the resolved state of a discussion + """ + discussionToggleResolve(input: DiscussionToggleResolveInput!): DiscussionToggleResolvePayload dismissVulnerability(input: DismissVulnerabilityInput!): DismissVulnerabilityPayload epicAddIssue(input: EpicAddIssueInput!): EpicAddIssuePayload epicSetSubscription(input: EpicSetSubscriptionInput!): EpicSetSubscriptionPayload @@ -6675,7 +7588,9 @@ type Mutation { issueSetIteration(input: IssueSetIterationInput!): IssueSetIterationPayload issueSetWeight(input: IssueSetWeightInput!): IssueSetWeightPayload jiraImportStart(input: JiraImportStartInput!): JiraImportStartPayload + jiraImportUsers(input: JiraImportUsersInput!): JiraImportUsersPayload markAsSpamSnippet(input: MarkAsSpamSnippetInput!): MarkAsSpamSnippetPayload + mergeRequestCreate(input: MergeRequestCreateInput!): MergeRequestCreatePayload mergeRequestSetAssignees(input: MergeRequestSetAssigneesInput!): MergeRequestSetAssigneesPayload mergeRequestSetLabels(input: MergeRequestSetLabelsInput!): MergeRequestSetLabelsPayload mergeRequestSetLocked(input: MergeRequestSetLockedInput!): MergeRequestSetLockedPayload @@ -6684,12 +7599,14 @@ type Mutation { mergeRequestSetWip(input: MergeRequestSetWipInput!): MergeRequestSetWipPayload removeAwardEmoji(input: RemoveAwardEmojiInput!): RemoveAwardEmojiPayload removeProjectFromSecurityDashboard(input: RemoveProjectFromSecurityDashboardInput!): RemoveProjectFromSecurityDashboardPayload + runDastScan(input: RunDASTScanInput!): RunDASTScanPayload todoMarkDone(input: TodoMarkDoneInput!): TodoMarkDonePayload todoRestore(input: TodoRestoreInput!): TodoRestorePayload todoRestoreMany(input: TodoRestoreManyInput!): TodoRestoreManyPayload todosMarkAllDone(input: TodosMarkAllDoneInput!): TodosMarkAllDonePayload toggleAwardEmoji(input: ToggleAwardEmojiInput!): ToggleAwardEmojiPayload updateAlertStatus(input: UpdateAlertStatusInput!): UpdateAlertStatusPayload + updateContainerExpirationPolicy(input: UpdateContainerExpirationPolicyInput!): UpdateContainerExpirationPolicyPayload updateEpic(input: UpdateEpicInput!): UpdateEpicPayload """ @@ -6856,7 +7773,7 @@ type NamespaceEdge { node: Namespace } -type Note { +type Note implements ResolvableInterface { """ User who wrote this note """ @@ -6903,17 +7820,22 @@ type Note { project: Project """ - Indicates if this note can be resolved. That is, if it is a resolvable discussion or simply a standalone note + Indicates if the object can be resolved """ resolvable: Boolean! """ - Timestamp of the note's resolution + Indicates if the object is resolved + """ + resolved: Boolean! + + """ + Timestamp of when the object was resolved """ resolvedAt: Time """ - User that resolved the discussion + User who resolved the object """ resolvedBy: User @@ -7301,6 +8223,11 @@ type Pipeline { iid: String! """ + Vulnerability and scanned resource counts for each security scanner of the pipeline + """ + securityReportSummary: SecurityReportSummary + + """ SHA of the pipeline's commit """ sha: String! @@ -7422,7 +8349,12 @@ type Project { """ Counts of alerts by status for the project """ - alertManagementAlertStatusCounts: AlertManagementAlertStatusCountsType + alertManagementAlertStatusCounts( + """ + Search criteria for filtering alerts. This will search on title, description, service, monitoring_tool. + """ + search: String + ): AlertManagementAlertStatusCountsType """ Alert Management alerts of the project @@ -7470,6 +8402,12 @@ type Project { ): AlertManagementAlertConnection """ + If `only_allow_merge_if_pipeline_succeeds` is true, indicates if merge + requests of the project can also be merged with skipped jobs + """ + allowMergeOnSkippedPipeline: Boolean + + """ Indicates the archived status of the project """ archived: Boolean @@ -7525,6 +8463,11 @@ type Project { ): BoardConnection """ + The container expiration policy of the project + """ + containerExpirationPolicy: ContainerExpirationPolicy + + """ Indicates if the project stores Docker container images in a container registry """ containerRegistryEnabled: Boolean @@ -7840,6 +8783,46 @@ type Project { jobsEnabled: Boolean """ + A label available on this project + """ + label( + """ + Title of the label + """ + title: String! + ): Label + + """ + Labels available on this project + """ + labels( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + A search term to find labels with + """ + searchTerm: String + ): LabelConnection + + """ Timestamp of the project last activity """ lastActivityAt: Time @@ -7856,12 +8839,7 @@ type Project { """ IID of the merge request, for example `1` """ - iid: String - - """ - Array of IIDs of merge requests, for example `[1, 2]` - """ - iids: [String!] + iid: String! ): MergeRequest """ @@ -7884,19 +8862,34 @@ type Project { first: Int """ - IID of the merge request, for example `1` + Array of IIDs of merge requests, for example `[1, 2]` """ - iid: String + iids: [String!] """ - Array of IIDs of merge requests, for example `[1, 2]` + Array of label names. All resolved merge requests will have all of these labels. """ - iids: [String!] + labels: [String!] """ Returns the last _n_ elements from the list. """ last: Int + + """ + Array of source branch names. All resolved merge requests will have one of these branches as their source. + """ + sourceBranches: [String!] + + """ + A merge request state. If provided, all resolved merge requests will have this state. + """ + state: MergeRequestState + + """ + Array of target branch names. All resolved merge requests will have one of these branches as their target. + """ + targetBranches: [String!] ): MergeRequestConnection """ @@ -7972,6 +8965,16 @@ type Project { path: String! """ + Build pipeline of the project + """ + pipeline( + """ + IID of the Pipeline, e.g., "1" + """ + iid: ID! + ): Pipeline + + """ Build pipelines of the project """ pipelines( @@ -8018,6 +9021,36 @@ type Project { printingMergeRequestLinkEnabled: Boolean """ + Members of the project + """ + projectMembers( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Search query + """ + search: String + ): ProjectMemberConnection + + """ Indicates if there is public access to pipelines and job details of the project, including output logs and artifacts """ publicJobs: Boolean @@ -8077,6 +9110,11 @@ type Project { """ requirement( """ + Filter requirements by author username + """ + authorUsername: [String!] + + """ IID of the requirement, e.g., "1" """ iid: ID @@ -8087,6 +9125,11 @@ type Project { iids: [ID!] """ + Filter requirements by title search + """ + search: String + + """ List requirements by sort order """ sort: Sort @@ -8112,6 +9155,11 @@ type Project { after: String """ + Filter requirements by author username + """ + authorUsername: [String!] + + """ Returns the elements in the list that come before the specified cursor. """ before: String @@ -8137,6 +9185,11 @@ type Project { last: Int """ + Filter requirements by title search + """ + search: String + + """ List requirements by sort order """ sort: Sort @@ -8383,6 +9436,91 @@ type ProjectEdge { node: Project } +""" +Represents a Project Member +""" +type ProjectMember implements MemberInterface { + """ + GitLab::Access level + """ + accessLevel: AccessLevel + + """ + Date and time the membership was created + """ + createdAt: Time + + """ + User that authorized membership + """ + createdBy: User + + """ + Date and time the membership expires + """ + expiresAt: Time + + """ + ID of the member + """ + id: ID! + + """ + Project that User is a member of + """ + project: Project + + """ + Date and time the membership was last updated + """ + updatedAt: Time + + """ + User that is associated with the member object + """ + user: User! + + """ + Permissions for the current user on the resource + """ + userPermissions: ProjectPermissions! +} + +""" +The connection type for ProjectMember. +""" +type ProjectMemberConnection { + """ + A list of edges. + """ + edges: [ProjectMemberEdge] + + """ + A list of nodes. + """ + nodes: [ProjectMember] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ProjectMemberEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: ProjectMember +} + type ProjectPermissions { """ Indicates the user can perform `admin_operations` on this resource @@ -8794,6 +9932,61 @@ type Query { ): SnippetConnection """ + Find a user on this instance + """ + user( + """ + ID of the User + """ + id: ID + + """ + Username of the User + """ + username: String + ): User + + """ + Find users + """ + users( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + List of user Global IDs + """ + ids: [ID!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Sort users by this criteria + """ + sort: Sort = created_desc + + """ + List of usernames + """ + usernames: [String!] + ): UserConnection + + """ Vulnerabilities reported on projects on the current user's instance security dashboard """ vulnerabilities( @@ -8901,6 +10094,11 @@ enum RegistryState { type Release { """ + Assets of the release + """ + assets: ReleaseAssets + + """ User that created the release """ author: User @@ -8926,6 +10124,31 @@ type Release { descriptionHtml: String """ + Evidence for the release + """ + evidences( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ReleaseEvidenceConnection + + """ Milestones associated to the release """ milestones( @@ -8971,6 +10194,63 @@ type Release { tagPath: String } +type ReleaseAssets { + """ + Number of assets of the release + """ + assetsCount: Int + + """ + Asset links of the release + """ + links( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ReleaseLinkConnection + + """ + Sources of the release + """ + sources( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ReleaseSourceConnection +} + """ The connection type for Release. """ @@ -9007,6 +10287,200 @@ type ReleaseEdge { } """ +Evidence for a release +""" +type ReleaseEvidence { + """ + Timestamp when the evidence was collected + """ + collectedAt: Time + + """ + URL from where the evidence can be downloaded + """ + filepath: String + + """ + ID of the evidence + """ + id: ID! + + """ + SHA1 ID of the evidence hash + """ + sha: String +} + +""" +The connection type for ReleaseEvidence. +""" +type ReleaseEvidenceConnection { + """ + A list of edges. + """ + edges: [ReleaseEvidenceEdge] + + """ + A list of nodes. + """ + nodes: [ReleaseEvidence] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ReleaseEvidenceEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: ReleaseEvidence +} + +type ReleaseLink { + """ + Indicates the link points to an external resource + """ + external: Boolean + + """ + ID of the link + """ + id: ID! + + """ + Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` + """ + linkType: ReleaseLinkType + + """ + Name of the link + """ + name: String + + """ + URL of the link + """ + url: String +} + +""" +The connection type for ReleaseLink. +""" +type ReleaseLinkConnection { + """ + A list of edges. + """ + edges: [ReleaseLinkEdge] + + """ + A list of nodes. + """ + nodes: [ReleaseLink] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ReleaseLinkEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: ReleaseLink +} + +""" +Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` +""" +enum ReleaseLinkType { + """ + Image link type + """ + IMAGE + + """ + Other link type + """ + OTHER + + """ + Package link type + """ + PACKAGE + + """ + Runbook link type + """ + RUNBOOK +} + +type ReleaseSource { + """ + Format of the source + """ + format: String + + """ + Download URL of the source + """ + url: String +} + +""" +The connection type for ReleaseSource. +""" +type ReleaseSourceConnection { + """ + A list of edges. + """ + edges: [ReleaseSourceEdge] + + """ + A list of nodes. + """ + nodes: [ReleaseSource] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type ReleaseSourceEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: ReleaseSource +} + +""" Autogenerated input type of RemoveAwardEmoji """ input RemoveAwardEmojiInput { @@ -9114,7 +10588,7 @@ type Repository { } """ -Represents a requirement. +Represents a requirement """ type Requirement { """ @@ -9148,6 +10622,36 @@ type Requirement { state: RequirementState! """ + Test reports of the requirement + """ + testReports( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + List test reports by sort order + """ + sort: Sort + ): TestReportConnection + + """ Title of the requirement """ title: String @@ -9251,6 +10755,28 @@ type RequirementStatesCount { opened: Int } +interface ResolvableInterface { + """ + Indicates if the object can be resolved + """ + resolvable: Boolean! + + """ + Indicates if the object is resolved + """ + resolved: Boolean! + + """ + Timestamp of when the object was resolved + """ + resolvedAt: Time + + """ + User who resolved the object + """ + resolvedBy: User +} + type RootStorageStatistics { """ The CI artifacts size in bytes @@ -9284,6 +10810,101 @@ type RootStorageStatistics { } """ +Autogenerated input type of RunDASTScan +""" +input RunDASTScanInput { + """ + The branch to be associated with the scan. + """ + branch: String! + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The project the DAST scan belongs to. + """ + projectPath: ID! + + """ + The type of scan to be run. + """ + scanType: DastScanTypeEnum! + + """ + The URL of the target to be scanned. + """ + targetUrl: String! +} + +""" +Autogenerated return type of RunDASTScan +""" +type RunDASTScanPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + URL of the pipeline that was created. + """ + pipelineUrl: String +} + +""" +Represents summary of a security report +""" +type SecurityReportSummary { + """ + Aggregated counts for the container_scanning scan + """ + containerScanning: SecurityReportSummarySection + + """ + Aggregated counts for the dast scan + """ + dast: SecurityReportSummarySection + + """ + Aggregated counts for the dependency_scanning scan + """ + dependencyScanning: SecurityReportSummarySection + + """ + Aggregated counts for the sast scan + """ + sast: SecurityReportSummarySection + + """ + Aggregated counts for the secret_detection scan + """ + secretDetection: SecurityReportSummarySection +} + +""" +Represents a section of a summary of a security report +""" +type SecurityReportSummarySection { + """ + Total number of scanned resources + """ + scannedResourcesCount: Int + + """ + Total number of vulnerabilities + """ + vulnerabilitiesCount: Int +} + +""" A Sentry error. """ type SentryDetailedError { @@ -9816,7 +11437,7 @@ type Snippet implements Noteable { """ The owner of the snippet """ - author: User! + author: User """ Snippet blob @@ -9824,6 +11445,11 @@ type Snippet implements Noteable { blob: SnippetBlob! """ + Snippet blobs + """ + blobs: [SnippetBlob!]! + + """ Timestamp this snippet was created """ createdAt: Time! @@ -10234,6 +11860,73 @@ type TaskCompletionStatus { } """ +Represents a requirement test report. +""" +type TestReport { + """ + Author of the test report + """ + author: User + + """ + Timestamp of when the test report was created + """ + createdAt: Time! + + """ + ID of the test report + """ + id: ID! + + """ + State of the test report + """ + state: TestReportState! +} + +""" +The connection type for TestReport. +""" +type TestReportConnection { + """ + A list of edges. + """ + edges: [TestReportEdge] + + """ + A list of nodes. + """ + nodes: [TestReport] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type TestReportEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: TestReport +} + +""" +State of a test report +""" +enum TestReportState { + PASSED +} + +""" Time represented in ISO 8601 """ scalar Time @@ -10819,6 +12512,61 @@ type UpdateAlertStatusPayload { issue: Issue } +""" +Autogenerated input type of UpdateContainerExpirationPolicy +""" +input UpdateContainerExpirationPolicyInput { + """ + This container expiration policy schedule + """ + cadence: ContainerExpirationPolicyCadenceEnum + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Indicates whether this container expiration policy is enabled + """ + enabled: Boolean + + """ + Number of tags to retain + """ + keepN: ContainerExpirationPolicyKeepEnum + + """ + Tags older that this will expire + """ + olderThan: ContainerExpirationPolicyOlderThanEnum + + """ + The project path where the container expiration policy is located + """ + projectPath: ID! +} + +""" +Autogenerated return type of UpdateContainerExpirationPolicy +""" +type UpdateContainerExpirationPolicyPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + The container expiration policy after mutation + """ + containerExpirationPolicy: ContainerExpirationPolicy + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! +} + input UpdateDiffImagePositionInput { """ Total height of the image @@ -11195,11 +12943,156 @@ scalar Upload type User { """ + Merge Requests assigned to the user + """ + assignedMergeRequests( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Array of IIDs of merge requests, for example `[1, 2]` + """ + iids: [String!] + + """ + Array of label names. All resolved merge requests will have all of these labels. + """ + labels: [String!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + The global ID of the project the authored merge requests should be in. Incompatible with projectPath. + """ + projectId: ID + + """ + The full-path of the project the authored merge requests should be in. Incompatible with projectId. + """ + projectPath: String + + """ + Array of source branch names. All resolved merge requests will have one of these branches as their source. + """ + sourceBranches: [String!] + + """ + A merge request state. If provided, all resolved merge requests will have this state. + """ + state: MergeRequestState + + """ + Array of target branch names. All resolved merge requests will have one of these branches as their target. + """ + targetBranches: [String!] + ): MergeRequestConnection + + """ + Merge Requests authored by the user + """ + authoredMergeRequests( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Array of IIDs of merge requests, for example `[1, 2]` + """ + iids: [String!] + + """ + Array of label names. All resolved merge requests will have all of these labels. + """ + labels: [String!] + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + The global ID of the project the authored merge requests should be in. Incompatible with projectPath. + """ + projectId: ID + + """ + The full-path of the project the authored merge requests should be in. Incompatible with projectId. + """ + projectPath: String + + """ + Array of source branch names. All resolved merge requests will have one of these branches as their source. + """ + sourceBranches: [String!] + + """ + A merge request state. If provided, all resolved merge requests will have this state. + """ + state: MergeRequestState + + """ + Array of target branch names. All resolved merge requests will have one of these branches as their target. + """ + targetBranches: [String!] + ): MergeRequestConnection + + """ URL of the user's avatar """ avatarUrl: String """ + Group memberships of the user + """ + groupMemberships( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): GroupMemberConnection + + """ ID of the user """ id: ID! @@ -11210,6 +13103,31 @@ type User { name: String! """ + Project memberships of the user + """ + projectMemberships( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ProjectMemberConnection + + """ Snippets authored by the user """ snippets( @@ -11250,9 +13168,9 @@ type User { ): SnippetConnection """ - State of the issue + State of the user """ - state: String! + state: UserState! """ Todos of the user @@ -11367,6 +13285,26 @@ type UserPermissions { createSnippet: Boolean! } +""" +Possible states of a user +""" +enum UserState { + """ + The user is active and is able to use the system + """ + active + + """ + The user has been blocked and is prevented from using the system + """ + blocked + + """ + The user is no longer active and is unable to use the system + """ + deactivated +} + enum VisibilityLevelsEnum { internal private @@ -11449,6 +13387,36 @@ type Vulnerability { id: ID! """ + List of issue links related to the vulnerability + """ + issueLinks( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + + """ + Filter issue links by link type + """ + linkType: VulnerabilityIssueLinkType + ): VulnerabilityIssueLinkConnection! + + """ Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability """ location: VulnerabilityLocation @@ -11459,7 +13427,8 @@ type Vulnerability { project: Project """ - Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST) + Type of the security report that found the vulnerability (SAST, + DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION) """ reportType: VulnerabilityReportType @@ -11479,6 +13448,11 @@ type Vulnerability { title: String """ + Number of user notes attached to the vulnerability + """ + userNotesCount: Int! + + """ Permissions for the current user on the resource """ userPermissions: VulnerabilityPermissions! @@ -11525,9 +13499,72 @@ type VulnerabilityEdge { } """ +Represents an issue link of a vulnerability. +""" +type VulnerabilityIssueLink { + """ + GraphQL ID of the vulnerability + """ + id: ID! + + """ + The issue attached to issue link + """ + issue: Issue! + + """ + Type of the issue link + """ + linkType: VulnerabilityIssueLinkType! +} + +""" +The connection type for VulnerabilityIssueLink. +""" +type VulnerabilityIssueLinkConnection { + """ + A list of edges. + """ + edges: [VulnerabilityIssueLinkEdge] + + """ + A list of nodes. + """ + nodes: [VulnerabilityIssueLink] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! +} + +""" +An edge in a connection. +""" +type VulnerabilityIssueLinkEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: VulnerabilityIssueLink +} + +""" +The type of the issue link related to a vulnerability. +""" +enum VulnerabilityIssueLinkType { + CREATED + RELATED +} + +""" Represents a vulnerability location. The fields with data will depend on the vulnerability report type """ -union VulnerabilityLocation = VulnerabilityLocationContainerScanning | VulnerabilityLocationDast | VulnerabilityLocationDependencyScanning | VulnerabilityLocationSast +union VulnerabilityLocation = VulnerabilityLocationContainerScanning | VulnerabilityLocationDast | VulnerabilityLocationDependencyScanning | VulnerabilityLocationSast | VulnerabilityLocationSecretDetection """ Represents the location of a vulnerability found by a container security scan @@ -11620,6 +13657,36 @@ type VulnerabilityLocationSast { } """ +Represents the location of a vulnerability found by a secret detection scan +""" +type VulnerabilityLocationSecretDetection { + """ + Number of the last relevant line in the vulnerable file + """ + endLine: String + + """ + Path to the vulnerable file + """ + file: String + + """ + Number of the first relevant line in the vulnerable file + """ + startLine: String + + """ + Class containing the vulnerability + """ + vulnerableClass: String + + """ + Method containing the vulnerability + """ + vulnerableMethod: String +} + +""" Check permissions for the current user on a vulnerability """ type VulnerabilityPermissions { @@ -11672,6 +13739,7 @@ enum VulnerabilityReportType { DAST DEPENDENCY_SCANNING SAST + SECRET_DETECTION } """ diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 40bfa08cff3..d2bc599ff9d 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -10,6 +10,94 @@ "subscriptionType": null, "types": [ { + "kind": "OBJECT", + "name": "AccessLevel", + "description": "Represents the access level of a relationship between a User and object that it is related to", + "fields": [ + { + "name": "integerValue", + "description": "Integer representation of access level", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "stringValue", + "description": "String representation of access level", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "AccessLevelEnum", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "AccessLevelEnum", + "description": "Access level to a resource", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "NO_ACCESS", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "GUEST", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "REPORTER", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DEVELOPER", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "MAINTAINER", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "OWNER", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "AddAwardEmojiInput", "description": "Autogenerated input type of AddAwardEmoji", @@ -395,6 +483,59 @@ "description": "Describes an alert from the project's Alert Management", "fields": [ { + "name": "assignees", + "description": "Assignees of the alert", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UserConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createdAt", "description": "Timestamp the alert was created", "args": [ @@ -437,6 +578,63 @@ "deprecationReason": null }, { + "name": "discussions", + "description": "All discussions on this noteable", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "DiscussionConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "endedAt", "description": "Timestamp the alert ended", "args": [ @@ -533,6 +731,63 @@ "deprecationReason": null }, { + "name": "notes", + "description": "All notes on this noteable", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "NoteConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "service", "description": "Service the alert came from", "args": [ @@ -619,7 +874,11 @@ ], "inputFields": null, "interfaces": [ - + { + "kind": "INTERFACE", + "name": "Noteable", + "ofType": null + } ], "enumValues": null, "possibleTypes": null @@ -769,25 +1028,25 @@ "deprecationReason": null }, { - "name": "START_TIME_ASC", + "name": "STARTED_AT_ASC", "description": "Start time by ascending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "START_TIME_DESC", + "name": "STARTED_AT_DESC", "description": "Start time by descending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "END_TIME_ASC", + "name": "ENDED_AT_ASC", "description": "End time by ascending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "END_TIME_DESC", + "name": "ENDED_AT_DESC", "description": "End time by descending order", "isDeprecated": false, "deprecationReason": null @@ -817,13 +1076,13 @@ "deprecationReason": null }, { - "name": "EVENTS_COUNT_ASC", + "name": "EVENT_COUNT_ASC", "description": "Events count by ascending order", "isDeprecated": false, "deprecationReason": null }, { - "name": "EVENTS_COUNT_DESC", + "name": "EVENT_COUNT_DESC", "description": "Events count by descending order", "isDeprecated": false, "deprecationReason": null @@ -1035,6 +1294,168 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "AlertSetAssigneesInput", + "description": "Autogenerated input type of AlertSetAssignees", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the alert to mutate is in", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The iid of the alert to mutate", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "assigneeUsernames", + "description": "The usernames to assign to the alert. Replaces existing assignees by default.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "defaultValue": null + }, + { + "name": "operationMode", + "description": "The operation to perform. Defaults to REPLACE.", + "type": { + "kind": "ENUM", + "name": "MutationOperationMode", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AlertSetAssigneesPayload", + "description": "Autogenerated return type of AlertSetAssignees", + "fields": [ + { + "name": "alert", + "description": "The alert after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue created after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "AwardEmoji", "description": "An emoji awarded by a user.", @@ -1519,7 +1940,7 @@ }, { "name": "lists", - "description": "Lists of the project board", + "description": "Lists of the board", "args": [ { "name": "after", @@ -2517,6 +2938,589 @@ }, { "kind": "INPUT_OBJECT", + "name": "CommitAction", + "description": null, + "fields": null, + "inputFields": [ + { + "name": "action", + "description": "The action to perform, create, delete, move, update, chmod", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "CommitActionMode", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "filePath", + "description": "Full path to the file", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "content", + "description": "Content of the file", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "previousPath", + "description": "Original full path to the file being moved", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "lastCommitId", + "description": "Last known file commit ID", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "executeFilemode", + "description": "Enables/disables the execute flag on the file", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "encoding", + "description": "Encoding of the file. Default is text", + "type": { + "kind": "ENUM", + "name": "CommitEncoding", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "CommitActionMode", + "description": "Mode of a commit action", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "CREATE", + "description": "Create command", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "DELETE", + "description": "Delete command", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "MOVE", + "description": "Move command", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "UPDATE", + "description": "Update command", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CHMOD", + "description": "Chmod command", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "CommitCreateInput", + "description": "Autogenerated input type of CommitCreate", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "Project full path the branch is associated with", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "branch", + "description": "Name of the branch", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "message", + "description": "Raw commit message", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "actions", + "description": "Array of action hashes to commit as a batch", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CommitAction", + "ofType": null + } + } + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CommitCreatePayload", + "description": "Autogenerated return type of CommitCreate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "commit", + "description": "The commit after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Commit", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "CommitEncoding", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "TEXT", + "description": "Text encoding", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "BASE64", + "description": "Base64 encoding", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ContainerExpirationPolicy", + "description": "A tag expiration policy designed to keep only the images that matter most", + "fields": [ + { + "name": "cadence", + "description": "This container expiration policy schedule", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "ContainerExpirationPolicyCadenceEnum", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdAt", + "description": "Timestamp of when the container expiration policy was created", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "enabled", + "description": "Indicates whether this container expiration policy is enabled", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "keepN", + "description": "Number of tags to retain", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "ContainerExpirationPolicyKeepEnum", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nameRegex", + "description": "Tags with names matching this regex pattern will expire", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nameRegexKeep", + "description": "Tags with names matching this regex pattern will be preserved", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nextRunAt", + "description": "Next time that this container expiration policy will get executed", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "olderThan", + "description": "Tags older that this will expire", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "ContainerExpirationPolicyOlderThanEnum", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Timestamp of when the container expiration policy was updated", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "ContainerExpirationPolicyCadenceEnum", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "EVERY_DAY", + "description": "Every day", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "EVERY_WEEK", + "description": "Every week", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "EVERY_TWO_WEEKS", + "description": "Every two weeks", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "EVERY_MONTH", + "description": "Every month", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "EVERY_THREE_MONTHS", + "description": "Every three months", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "ContainerExpirationPolicyKeepEnum", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "ONE_TAG", + "description": "1 tag per image name", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "FIVE_TAGS", + "description": "5 tags per image name", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "TEN_TAGS", + "description": "10 tags per image name", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "TWENTY_FIVE_TAGS", + "description": "25 tags per image name", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "FIFTY_TAGS", + "description": "50 tags per image name", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "ONE_HUNDRED_TAGS", + "description": "100 tags per image name", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "ContainerExpirationPolicyOlderThanEnum", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "SEVEN_DAYS", + "description": "7 days until tags are automatically removed", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "FOURTEEN_DAYS", + "description": "14 days until tags are automatically removed", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "THIRTY_DAYS", + "description": "30 days until tags are automatically removed", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "NINETY_DAYS", + "description": "90 days until tags are automatically removed", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateAlertIssueInput", "description": "Autogenerated input type of CreateAlertIssue", "fields": null, @@ -3966,6 +4970,111 @@ "possibleTypes": null }, { + "kind": "ENUM", + "name": "DastScanTypeEnum", + "description": null, + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "PASSIVE", + "description": "Passive DAST scan. This scan will not make active attacks against the target site.", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DeleteAnnotationInput", + "description": "Autogenerated input type of DeleteAnnotation", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "The global id of the annotation to delete", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DeleteAnnotationPayload", + "description": "Autogenerated return type of DeleteAnnotation", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "DeleteJobsResponse", "description": "The response from the AdminSidekiqQueuesDeleteJobs mutation.", @@ -7134,11 +8243,79 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "resolvable", + "description": "Indicates if the object can be resolved", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolved", + "description": "Indicates if the object is resolved", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolvedAt", + "description": "Timestamp of when the object was resolved", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolvedBy", + "description": "User who resolved the object", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, "interfaces": [ - + { + "kind": "INTERFACE", + "name": "ResolvableInterface", + "ofType": null + } ], "enumValues": null, "possibleTypes": null @@ -7257,6 +8434,122 @@ }, { "kind": "INPUT_OBJECT", + "name": "DiscussionToggleResolveInput", + "description": "Autogenerated input type of DiscussionToggleResolve", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": "The global id of the discussion", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "resolve", + "description": "Will resolve the discussion when true, and unresolve the discussion when false", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DiscussionToggleResolvePayload", + "description": "Autogenerated return type of DiscussionToggleResolve", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "discussion", + "description": "The discussion after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Discussion", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "DismissVulnerabilityInput", "description": "Autogenerated input type of DismissVulnerability", "fields": null, @@ -12032,6 +13325,96 @@ "deprecationReason": null }, { + "name": "label", + "description": "A label available on this group", + "args": [ + { + "name": "title", + "description": "Title of the label", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Label", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "labels", + "description": "Labels available on this group", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "searchTerm", + "description": "A search term to find labels with", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "LabelConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "lfsEnabled", "description": "Indicates if Large File Storage (LFS) is enabled for namespace", "args": [ @@ -12740,6 +14123,237 @@ }, { "kind": "OBJECT", + "name": "GroupMember", + "description": "Represents a Group Member", + "fields": [ + { + "name": "accessLevel", + "description": "GitLab::Access level", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AccessLevel", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdAt", + "description": "Date and time the membership was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdBy", + "description": "User that authorized membership", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "expiresAt", + "description": "Date and time the membership expires", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "group", + "description": "Group that a User is a member of", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Group", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Date and time the membership was last updated", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "userPermissions", + "description": "Permissions for the current user on the resource", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "GroupPermissions", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "MemberInterface", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "GroupMemberConnection", + "description": "The connection type for GroupMember.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "GroupMemberEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "GroupMember", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "GroupMemberEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "GroupMember", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "GroupPermissions", "description": null, "fields": [ @@ -15029,6 +16643,42 @@ "deprecationReason": null }, { + "name": "failedToImportCount", + "description": "Count of issues that failed to import", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "importedIssuesCount", + "description": "Count of issues that were successfully imported", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "jiraProjectKey", "description": "Project key for the imported Jira project", "args": [ @@ -15073,6 +16723,24 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "totalIssueCount", + "description": "Total count of issues that were attempted to import", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -15321,6 +16989,301 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "JiraImportUsersInput", + "description": "Autogenerated input type of JiraImportUsers", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project to import the Jira users into", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "startAt", + "description": "The index of the record the import should started at, default 0 (50 records returned)", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "JiraImportUsersPayload", + "description": "Autogenerated return type of JiraImportUsers", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "jiraUsers", + "description": "Users returned from Jira, matched by email and name if possible.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "JiraUser", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "JiraProject", + "description": null, + "fields": [ + { + "name": "key", + "description": "Key of the Jira project", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the Jira project", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "projectId", + "description": "ID of the Jira project", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "JiraProjectConnection", + "description": "The connection type for JiraProject.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "JiraProjectEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "JiraProject", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "JiraProjectEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "JiraProject", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "JiraService", "description": null, @@ -15340,6 +17303,69 @@ "deprecationReason": null }, { + "name": "projects", + "description": "List of Jira projects fetched through Jira REST API", + "args": [ + { + "name": "name", + "description": "Project name or key", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "JiraProjectConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "type", "description": "Class name of the service", "args": [ @@ -15367,6 +17393,83 @@ }, { "kind": "OBJECT", + "name": "JiraUser", + "description": null, + "fields": [ + { + "name": "gitlabId", + "description": "Id of the matched GitLab user", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "jiraAccountId", + "description": "Account id of the Jira user", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "jiraDisplayName", + "description": "Display name of the Jira user", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "jiraEmail", + "description": "Email of the Jira user, returned only for users with public emails", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "Label", "description": null, "fields": [ @@ -15722,6 +17825,98 @@ "possibleTypes": null }, { + "kind": "INTERFACE", + "name": "MemberInterface", + "description": null, + "fields": [ + { + "name": "accessLevel", + "description": "GitLab::Access level", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AccessLevel", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdAt", + "description": "Date and time the membership was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdBy", + "description": "User that authorized membership", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "expiresAt", + "description": "Date and time the membership expires", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Date and time the membership was last updated", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": [ + { + "kind": "OBJECT", + "name": "GroupMember", + "ofType": null + }, + { + "kind": "OBJECT", + "name": "ProjectMember", + "ofType": null + } + ] + }, + { "kind": "OBJECT", "name": "MergeRequest", "description": null, @@ -15794,6 +17989,20 @@ "deprecationReason": null }, { + "name": "author", + "description": "User who created this merge request", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createdAt", "description": "Timestamp of when the merge request was created", "args": [ @@ -16208,6 +18417,20 @@ "deprecationReason": null }, { + "name": "mergedAt", + "description": "Timestamp of when the merge request was merged, null if not merged", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "milestone", "description": "The milestone of the merge request", "args": [ @@ -16664,6 +18887,24 @@ "deprecationReason": null }, { + "name": "targetBranchExists", + "description": "Indicates if the target branch of the merge request exists", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "targetProject", "description": "Target project of the merge request", "args": [ @@ -16965,6 +19206,160 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "MergeRequestCreateInput", + "description": "Autogenerated input type of MergeRequestCreate", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "Project full path the merge request is associated with", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "title", + "description": "Title of the merge request", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "sourceBranch", + "description": "Source branch of the merge request", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "targetBranch", + "description": "Target branch of the merge request", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "description", + "description": "Description of the merge request (Markdown rendered as HTML for caching)", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "MergeRequestCreatePayload", + "description": "Autogenerated return type of MergeRequestCreate", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "mergeRequest", + "description": "The merge request after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequest", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "MergeRequestEdge", "description": "An edge in a connection.", @@ -18157,6 +20552,28 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "schemaValidationWarnings", + "description": "Dashboard schema validation warnings", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, @@ -18417,6 +20834,24 @@ "deprecationReason": null }, { + "name": "groupMilestone", + "description": "Indicates if milestone is at group level", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "id", "description": "ID of the milestone", "args": [ @@ -18435,6 +20870,24 @@ "deprecationReason": null }, { + "name": "projectMilestone", + "description": "Indicates if milestone is at project level", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "startDate", "description": "Timestamp of the milestone start date", "args": [ @@ -18467,6 +20920,24 @@ "deprecationReason": null }, { + "name": "subgroupMilestone", + "description": "Indicates if milestone is at subgroup level", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "title", "description": "Title of the milestone", "args": [ @@ -18773,6 +21244,33 @@ "deprecationReason": null }, { + "name": "alertSetAssignees", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "AlertSetAssigneesInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AlertSetAssigneesPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "boardListUpdateLimitMetrics", "description": null, "args": [ @@ -18800,6 +21298,33 @@ "deprecationReason": null }, { + "name": "commitCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CommitCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CommitCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createAlertIssue", "description": null, "args": [ @@ -19070,6 +21595,33 @@ "deprecationReason": null }, { + "name": "deleteAnnotation", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DeleteAnnotationInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DeleteAnnotationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "designManagementDelete", "description": null, "args": [ @@ -19178,6 +21730,33 @@ "deprecationReason": null }, { + "name": "discussionToggleResolve", + "description": "Toggles the resolved state of a discussion", + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DiscussionToggleResolveInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DiscussionToggleResolvePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "dismissVulnerability", "description": null, "args": [ @@ -19421,6 +22000,33 @@ "deprecationReason": null }, { + "name": "jiraImportUsers", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "JiraImportUsersInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "JiraImportUsersPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "markAsSpamSnippet", "description": null, "args": [ @@ -19448,6 +22054,33 @@ "deprecationReason": null }, { + "name": "mergeRequestCreate", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "MergeRequestCreateInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequestCreatePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "mergeRequestSetAssignees", "description": null, "args": [ @@ -19664,6 +22297,33 @@ "deprecationReason": null }, { + "name": "runDastScan", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "RunDASTScanInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "RunDASTScanPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "todoMarkDone", "description": null, "args": [ @@ -19826,6 +22486,33 @@ "deprecationReason": null }, { + "name": "updateContainerExpirationPolicy", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "UpdateContainerExpirationPolicyInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UpdateContainerExpirationPolicyPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "updateEpic", "description": null, "args": [ @@ -20549,7 +23236,25 @@ }, { "name": "resolvable", - "description": "Indicates if this note can be resolved. That is, if it is a resolvable discussion or simply a standalone note", + "description": "Indicates if the object can be resolved", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolved", + "description": "Indicates if the object is resolved", "args": [ ], @@ -20567,7 +23272,7 @@ }, { "name": "resolvedAt", - "description": "Timestamp of the note's resolution", + "description": "Timestamp of when the object was resolved", "args": [ ], @@ -20581,7 +23286,7 @@ }, { "name": "resolvedBy", - "description": "User that resolved the discussion", + "description": "User who resolved the object", "args": [ ], @@ -20650,7 +23355,11 @@ ], "inputFields": null, "interfaces": [ - + { + "kind": "INTERFACE", + "name": "ResolvableInterface", + "ofType": null + } ], "enumValues": null, "possibleTypes": null @@ -20996,6 +23705,11 @@ "possibleTypes": [ { "kind": "OBJECT", + "name": "AlertManagementAlert", + "ofType": null + }, + { + "kind": "OBJECT", "name": "Design", "ofType": null }, @@ -21772,6 +24486,20 @@ "deprecationReason": null }, { + "name": "securityReportSummary", + "description": "Vulnerability and scanned resource counts for each security scanner of the pipeline", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummary", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "sha", "description": "SHA of the pipeline's commit", "args": [ @@ -22191,7 +24919,16 @@ "name": "alertManagementAlertStatusCounts", "description": "Counts of alerts by status for the project", "args": [ - + { + "name": "search", + "description": "Search criteria for filtering alerts. This will search on title, description, service, monitoring_tool.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } ], "type": { "kind": "OBJECT", @@ -22303,6 +25040,20 @@ "deprecationReason": null }, { + "name": "allowMergeOnSkippedPipeline", + "description": "If `only_allow_merge_if_pipeline_succeeds` is true, indicates if merge requests of the project can also be merged with skipped jobs", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "archived", "description": "Indicates the archived status of the project", "args": [ @@ -22431,6 +25182,20 @@ "deprecationReason": null }, { + "name": "containerExpirationPolicy", + "description": "The container expiration policy of the project", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ContainerExpirationPolicy", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "containerRegistryEnabled", "description": "Indicates if the project stores Docker container images in a container registry", "args": [ @@ -23181,6 +25946,96 @@ "deprecationReason": null }, { + "name": "label", + "description": "A label available on this project", + "args": [ + { + "name": "title", + "description": "Title of the label", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Label", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "labels", + "description": "Labels available on this project", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "searchTerm", + "description": "A search term to find labels with", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "LabelConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "lastActivityAt", "description": "Timestamp of the project last activity", "args": [ @@ -23216,26 +26071,12 @@ "name": "iid", "description": "IID of the merge request, for example `1`", "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "iids", - "description": "Array of IIDs of merge requests, for example `[1, 2]`", - "type": { - "kind": "LIST", + "kind": "NON_NULL", "name": null, "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } + "kind": "SCALAR", + "name": "String", + "ofType": null } }, "defaultValue": null @@ -23254,18 +26095,72 @@ "description": "Merge requests of the project", "args": [ { - "name": "iid", - "description": "IID of the merge request, for example `1`", + "name": "iids", + "description": "Array of IIDs of merge requests, for example `[1, 2]`", "type": { - "kind": "SCALAR", - "name": "String", + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "sourceBranches", + "description": "Array of source branch names. All resolved merge requests will have one of these branches as their source.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "targetBranches", + "description": "Array of target branch names. All resolved merge requests will have one of these branches as their target.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "A merge request state. If provided, all resolved merge requests will have this state.", + "type": { + "kind": "ENUM", + "name": "MergeRequestState", "ofType": null }, "defaultValue": null }, { - "name": "iids", - "description": "Array of IIDs of merge requests, for example `[1, 2]`", + "name": "labels", + "description": "Array of label names. All resolved merge requests will have all of these labels.", "type": { "kind": "LIST", "name": null, @@ -23522,6 +26417,33 @@ "deprecationReason": null }, { + "name": "pipeline", + "description": "Build pipeline of the project", + "args": [ + { + "name": "iid", + "description": "IID of the Pipeline, e.g., \"1\"", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "Pipeline", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "pipelines", "description": "Build pipelines of the project", "args": [ @@ -23619,6 +26541,69 @@ "deprecationReason": null }, { + "name": "projectMembers", + "description": "Members of the project", + "args": [ + { + "name": "search", + "description": "Search query", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ProjectMemberConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "publicJobs", "description": "Indicates if there is public access to pipelines and job details of the project, including output logs and artifacts", "args": [ @@ -23805,6 +26790,34 @@ "ofType": null }, "defaultValue": null + }, + { + "name": "search", + "description": "Filter requirements by title search", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "authorUsername", + "description": "Filter requirements by author username", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null } ], "type": { @@ -23882,6 +26895,34 @@ "defaultValue": null }, { + "name": "search", + "description": "Filter requirements by title search", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "authorUsername", + "description": "Filter requirements by author username", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { "name": "after", "description": "Returns the elements in the list that come after the specified cursor.", "type": { @@ -24576,6 +27617,273 @@ }, { "kind": "OBJECT", + "name": "ProjectMember", + "description": "Represents a Project Member", + "fields": [ + { + "name": "accessLevel", + "description": "GitLab::Access level", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "AccessLevel", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdAt", + "description": "Date and time the membership was created", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdBy", + "description": "User that authorized membership", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "expiresAt", + "description": "Date and time the membership expires", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the member", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "project", + "description": "Project that User is a member of", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Project", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updatedAt", + "description": "Date and time the membership was last updated", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "user", + "description": "User that is associated with the member object", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "User", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "userPermissions", + "description": "Permissions for the current user on the resource", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ProjectPermissions", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "MemberInterface", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ProjectMemberConnection", + "description": "The connection type for ProjectMember.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ProjectMemberEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ProjectMember", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ProjectMemberEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ProjectMember", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "ProjectPermissions", "description": null, "fields": [ @@ -25873,6 +29181,138 @@ "deprecationReason": null }, { + "name": "user", + "description": "Find a user on this instance", + "args": [ + { + "name": "id", + "description": "ID of the User", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "username", + "description": "Username of the User", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "users", + "description": "Find users", + "args": [ + { + "name": "ids", + "description": "List of user Global IDs", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "usernames", + "description": "List of usernames", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "sort", + "description": "Sort users by this criteria", + "type": { + "kind": "ENUM", + "name": "Sort", + "ofType": null + }, + "defaultValue": "created_desc" + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UserConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "vulnerabilities", "description": "Vulnerabilities reported on projects on the current user's instance security dashboard", "args": [ @@ -26127,6 +29567,20 @@ "description": null, "fields": [ { + "name": "assets", + "description": "Assets of the release", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseAssets", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "author", "description": "User that created the release", "args": [ @@ -26197,6 +29651,59 @@ "deprecationReason": null }, { + "name": "evidences", + "description": "Evidence for the release", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseEvidenceConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "milestones", "description": "Milestones associated to the release", "args": [ @@ -26319,6 +29826,139 @@ }, { "kind": "OBJECT", + "name": "ReleaseAssets", + "description": null, + "fields": [ + { + "name": "assetsCount", + "description": "Number of assets of the release", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "links", + "description": "Asset links of the release", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseLinkConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "sources", + "description": "Sources of the release", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseSourceConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "ReleaseConnection", "description": "The connection type for Release.", "fields": [ @@ -26430,6 +30070,578 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "ReleaseEvidence", + "description": "Evidence for a release", + "fields": [ + { + "name": "collectedAt", + "description": "Timestamp when the evidence was collected", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "filepath", + "description": "URL from where the evidence can be downloaded", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the evidence", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "sha", + "description": "SHA1 ID of the evidence hash", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseEvidenceConnection", + "description": "The connection type for ReleaseEvidence.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseEvidenceEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseEvidence", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseEvidenceEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseEvidence", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseLink", + "description": null, + "fields": [ + { + "name": "external", + "description": "Indicates the link points to an external resource", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the link", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "linkType", + "description": "Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`", + "args": [ + + ], + "type": { + "kind": "ENUM", + "name": "ReleaseLinkType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "name", + "description": "Name of the link", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "url", + "description": "URL of the link", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseLinkConnection", + "description": "The connection type for ReleaseLink.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseLinkEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseLink", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseLinkEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseLink", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "ReleaseLinkType", + "description": "Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "OTHER", + "description": "Other link type", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "RUNBOOK", + "description": "Runbook link type", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "PACKAGE", + "description": "Package link type", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "IMAGE", + "description": "Image link type", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseSource", + "description": null, + "fields": [ + { + "name": "format", + "description": "Format of the source", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "url", + "description": "Download URL of the source", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseSourceConnection", + "description": "The connection type for ReleaseSource.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseSourceEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "ReleaseSource", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "ReleaseSourceEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ReleaseSource", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "INPUT_OBJECT", "name": "RemoveAwardEmojiInput", "description": "Autogenerated input type of RemoveAwardEmoji", @@ -26742,7 +30954,7 @@ { "kind": "OBJECT", "name": "Requirement", - "description": "Represents a requirement.", + "description": "Represents a requirement", "fields": [ { "name": "author", @@ -26853,6 +31065,69 @@ "deprecationReason": null }, { + "name": "testReports", + "description": "Test reports of the requirement", + "args": [ + { + "name": "sort", + "description": "List test reports by sort order", + "type": { + "kind": "ENUM", + "name": "Sort", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "TestReportConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "title", "description": "Title of the requirement", "args": [ @@ -27190,6 +31465,92 @@ "possibleTypes": null }, { + "kind": "INTERFACE", + "name": "ResolvableInterface", + "description": null, + "fields": [ + { + "name": "resolvable", + "description": "Indicates if the object can be resolved", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolved", + "description": "Indicates if the object is resolved", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolvedAt", + "description": "Timestamp of when the object was resolved", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "resolvedBy", + "description": "User who resolved the object", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": [ + { + "kind": "OBJECT", + "name": "Discussion", + "ofType": null + }, + { + "kind": "OBJECT", + "name": "Note", + "ofType": null + } + ] + }, + { "kind": "OBJECT", "name": "RootStorageStatistics", "description": null, @@ -27311,6 +31672,274 @@ "possibleTypes": null }, { + "kind": "INPUT_OBJECT", + "name": "RunDASTScanInput", + "description": "Autogenerated input type of RunDASTScan", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project the DAST scan belongs to.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "targetUrl", + "description": "The URL of the target to be scanned.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "branch", + "description": "The branch to be associated with the scan.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "scanType", + "description": "The type of scan to be run.", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "DastScanTypeEnum", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RunDASTScanPayload", + "description": "Autogenerated return type of RunDASTScan", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pipelineUrl", + "description": "URL of the pipeline that was created.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SecurityReportSummary", + "description": "Represents summary of a security report", + "fields": [ + { + "name": "containerScanning", + "description": "Aggregated counts for the container_scanning scan", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dast", + "description": "Aggregated counts for the dast scan", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dependencyScanning", + "description": "Aggregated counts for the dependency_scanning scan", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "sast", + "description": "Aggregated counts for the sast scan", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "secretDetection", + "description": "Aggregated counts for the secret_detection scan", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "SecurityReportSummarySection", + "description": "Represents a section of a summary of a security report", + "fields": [ + { + "name": "scannedResourcesCount", + "description": "Total number of scanned resources", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerabilitiesCount", + "description": "Total number of vulnerabilities", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "SentryDetailedError", "description": "A Sentry error.", @@ -29085,11 +33714,25 @@ ], "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "blob", + "description": "Snippet blob", + "args": [ + + ], + "type": { "kind": "NON_NULL", "name": null, "ofType": { "kind": "OBJECT", - "name": "User", + "name": "SnippetBlob", "ofType": null } }, @@ -29097,8 +33740,8 @@ "deprecationReason": null }, { - "name": "blob", - "description": "Snippet blob", + "name": "blobs", + "description": "Snippet blobs", "args": [ ], @@ -29106,9 +33749,17 @@ "kind": "NON_NULL", "name": null, "ofType": { - "kind": "OBJECT", - "name": "SnippetBlob", - "ofType": null + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "SnippetBlob", + "ofType": null + } + } } }, "isDeprecated": false, @@ -30397,6 +35048,216 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "TestReport", + "description": "Represents a requirement test report.", + "fields": [ + { + "name": "author", + "description": "Author of the test report", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "createdAt", + "description": "Timestamp of when the test report was created", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": "ID of the test report", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state", + "description": "State of the test report", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "TestReportState", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "TestReportConnection", + "description": "The connection type for TestReport.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TestReportEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TestReport", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "TestReportEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "TestReport", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "TestReportState", + "description": "State of a test report", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "PASSED", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "SCALAR", "name": "Time", "description": "Time represented in ISO 8601", @@ -32195,6 +37056,148 @@ }, { "kind": "INPUT_OBJECT", + "name": "UpdateContainerExpirationPolicyInput", + "description": "Autogenerated input type of UpdateContainerExpirationPolicy", + "fields": null, + "inputFields": [ + { + "name": "projectPath", + "description": "The project path where the container expiration policy is located", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "enabled", + "description": "Indicates whether this container expiration policy is enabled", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "cadence", + "description": "This container expiration policy schedule", + "type": { + "kind": "ENUM", + "name": "ContainerExpirationPolicyCadenceEnum", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "olderThan", + "description": "Tags older that this will expire", + "type": { + "kind": "ENUM", + "name": "ContainerExpirationPolicyOlderThanEnum", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "keepN", + "description": "Number of tags to retain", + "type": { + "kind": "ENUM", + "name": "ContainerExpirationPolicyKeepEnum", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "UpdateContainerExpirationPolicyPayload", + "description": "Autogenerated return type of UpdateContainerExpirationPolicy", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "containerExpirationPolicy", + "description": "The container expiration policy after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "ContainerExpirationPolicy", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "UpdateDiffImagePositionInput", "description": null, "fields": null, @@ -33184,6 +38187,316 @@ "description": null, "fields": [ { + "name": "assignedMergeRequests", + "description": "Merge Requests assigned to the user", + "args": [ + { + "name": "iids", + "description": "Array of IIDs of merge requests, for example `[1, 2]`", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "sourceBranches", + "description": "Array of source branch names. All resolved merge requests will have one of these branches as their source.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "targetBranches", + "description": "Array of target branch names. All resolved merge requests will have one of these branches as their target.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "A merge request state. If provided, all resolved merge requests will have this state.", + "type": { + "kind": "ENUM", + "name": "MergeRequestState", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labels", + "description": "Array of label names. All resolved merge requests will have all of these labels.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "projectPath", + "description": "The full-path of the project the authored merge requests should be in. Incompatible with projectId.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "projectId", + "description": "The global ID of the project the authored merge requests should be in. Incompatible with projectPath.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequestConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "authoredMergeRequests", + "description": "Merge Requests authored by the user", + "args": [ + { + "name": "iids", + "description": "Array of IIDs of merge requests, for example `[1, 2]`", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "sourceBranches", + "description": "Array of source branch names. All resolved merge requests will have one of these branches as their source.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "targetBranches", + "description": "Array of target branch names. All resolved merge requests will have one of these branches as their target.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "state", + "description": "A merge request state. If provided, all resolved merge requests will have this state.", + "type": { + "kind": "ENUM", + "name": "MergeRequestState", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labels", + "description": "Array of label names. All resolved merge requests will have all of these labels.", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "projectPath", + "description": "The full-path of the project the authored merge requests should be in. Incompatible with projectId.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "projectId", + "description": "The global ID of the project the authored merge requests should be in. Incompatible with projectPath.", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "MergeRequestConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "avatarUrl", "description": "URL of the user's avatar", "args": [ @@ -33198,6 +38511,59 @@ "deprecationReason": null }, { + "name": "groupMemberships", + "description": "Group memberships of the user", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "GroupMemberConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "id", "description": "ID of the user", "args": [ @@ -33234,6 +38600,59 @@ "deprecationReason": null }, { + "name": "projectMemberships", + "description": "Project memberships of the user", + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ProjectMemberConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "snippets", "description": "Snippets authored by the user", "args": [ @@ -33326,7 +38745,7 @@ }, { "name": "state", - "description": "State of the issue", + "description": "State of the user", "args": [ ], @@ -33334,8 +38753,8 @@ "kind": "NON_NULL", "name": null, "ofType": { - "kind": "SCALAR", - "name": "String", + "kind": "ENUM", + "name": "UserState", "ofType": null } }, @@ -33714,6 +39133,35 @@ }, { "kind": "ENUM", + "name": "UserState", + "description": "Possible states of a user", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "active", + "description": "The user is active and is able to use the system", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "blocked", + "description": "The user has been blocked and is prevented from using the system", + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "deactivated", + "description": "The user is no longer active and is unable to use the system", + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { + "kind": "ENUM", "name": "VisibilityLevelsEnum", "description": null, "fields": null, @@ -33975,6 +39423,73 @@ "deprecationReason": null }, { + "name": "issueLinks", + "description": "List of issue links related to the vulnerability", + "args": [ + { + "name": "linkType", + "description": "Filter issue links by link type", + "type": { + "kind": "ENUM", + "name": "VulnerabilityIssueLinkType", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilityIssueLinkConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "location", "description": "Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability", "args": [ @@ -34004,7 +39519,7 @@ }, { "name": "reportType", - "description": "Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST)", + "description": "Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION)", "args": [ ], @@ -34059,6 +39574,24 @@ "deprecationReason": null }, { + "name": "userNotesCount", + "description": "Number of user notes attached to the vulnerability", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "userPermissions", "description": "Permissions for the current user on the resource", "args": [ @@ -34211,6 +39744,208 @@ "possibleTypes": null }, { + "kind": "OBJECT", + "name": "VulnerabilityIssueLink", + "description": "Represents an issue link of a vulnerability.", + "fields": [ + { + "name": "id", + "description": "GraphQL ID of the vulnerability", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue attached to issue link", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "linkType", + "description": "Type of the issue link", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "ENUM", + "name": "VulnerabilityIssueLinkType", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityIssueLinkConnection", + "description": "The connection type for VulnerabilityIssueLink.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilityIssueLinkEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "VulnerabilityIssueLink", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityIssueLinkEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "VulnerabilityIssueLink", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "ENUM", + "name": "VulnerabilityIssueLinkType", + "description": "The type of the issue link related to a vulnerability.", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": [ + { + "name": "RELATED", + "description": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "CREATED", + "description": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "possibleTypes": null + }, + { "kind": "UNION", "name": "VulnerabilityLocation", "description": "Represents a vulnerability location. The fields with data will depend on the vulnerability report type", @@ -34238,6 +39973,11 @@ "kind": "OBJECT", "name": "VulnerabilityLocationSast", "ofType": null + }, + { + "kind": "OBJECT", + "name": "VulnerabilityLocationSecretDetection", + "ofType": null } ] }, @@ -34491,6 +40231,89 @@ }, { "kind": "OBJECT", + "name": "VulnerabilityLocationSecretDetection", + "description": "Represents the location of a vulnerability found by a secret detection scan", + "fields": [ + { + "name": "endLine", + "description": "Number of the last relevant line in the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "file", + "description": "Path to the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "startLine", + "description": "Number of the first relevant line in the vulnerable file", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerableClass", + "description": "Class containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "vulnerableMethod", + "description": "Method containing the vulnerability", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", "name": "VulnerabilityPermissions", "description": "Check permissions for the current user on a vulnerability", "fields": [ @@ -34677,6 +40500,12 @@ "description": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "SECRET_DETECTION", + "description": null, + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 4164c26e751..befb57c1cba 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -16,6 +16,15 @@ fields and methods on a model are available via GraphQL. CAUTION: **Caution:** Fields that are deprecated are marked with **{warning-solid}**. +## AccessLevel + +Represents the access level of a relationship between a User and object that it is related to + +| Name | Type | Description | +| --- | ---- | ---------- | +| `integerValue` | Int | Integer representation of access level | +| `stringValue` | AccessLevelEnum | String representation of access level | + ## AddAwardEmojiPayload Autogenerated return type of AddAwardEmoji @@ -81,6 +90,17 @@ Represents total number of alerts for the represented categories | `resolved` | Int | Number of alerts with status RESOLVED for the project | | `triggered` | Int | Number of alerts with status TRIGGERED for the project | +## AlertSetAssigneesPayload + +Autogenerated return type of AlertSetAssignees + +| Name | Type | Description | +| --- | ---- | ---------- | +| `alert` | AlertManagementAlert | The alert after mutation | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue created after mutation | + ## AwardEmoji An emoji awarded by a user. @@ -177,6 +197,32 @@ Autogenerated return type of BoardListUpdateLimitMetrics | `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | | `webUrl` | String! | Web URL of the commit | +## CommitCreatePayload + +Autogenerated return type of CommitCreate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `commit` | Commit | The commit after mutation | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + +## ContainerExpirationPolicy + +A tag expiration policy designed to keep only the images that matter most + +| Name | Type | Description | +| --- | ---- | ---------- | +| `cadence` | ContainerExpirationPolicyCadenceEnum! | This container expiration policy schedule | +| `createdAt` | Time! | Timestamp of when the container expiration policy was created | +| `enabled` | Boolean! | Indicates whether this container expiration policy is enabled | +| `keepN` | ContainerExpirationPolicyKeepEnum | Number of tags to retain | +| `nameRegex` | String | Tags with names matching this regex pattern will expire | +| `nameRegexKeep` | String | Tags with names matching this regex pattern will be preserved | +| `nextRunAt` | Time | Next time that this container expiration policy will get executed | +| `olderThan` | ContainerExpirationPolicyOlderThanEnum | Tags older that this will expire | +| `updatedAt` | Time! | Timestamp of when the container expiration policy was updated | + ## CreateAlertIssuePayload Autogenerated return type of CreateAlertIssue @@ -278,6 +324,15 @@ Autogenerated return type of CreateSnippet | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `snippet` | Snippet | The snippet after mutation | +## DeleteAnnotationPayload + +Autogenerated return type of DeleteAnnotation + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ## DeleteJobsResponse The response from the AdminSidekiqQueuesDeleteJobs mutation. @@ -438,6 +493,20 @@ Autogenerated return type of DestroySnippet | `createdAt` | Time! | Timestamp of the discussion's creation | | `id` | ID! | ID of this discussion | | `replyId` | ID! | ID used to reply to this discussion | +| `resolvable` | Boolean! | Indicates if the object can be resolved | +| `resolved` | Boolean! | Indicates if the object is resolved | +| `resolvedAt` | Time | Timestamp of when the object was resolved | +| `resolvedBy` | User | User who resolved the object | + +## DiscussionToggleResolvePayload + +Autogenerated return type of DiscussionToggleResolve + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `discussion` | Discussion | The discussion after mutation | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | ## DismissVulnerabilityPayload @@ -665,6 +734,7 @@ Autogenerated return type of EpicTreeReorder | `fullPath` | ID! | Full path of the namespace | | `groupTimelogsEnabled` | Boolean | Indicates if Group timelogs are enabled for namespace | | `id` | ID! | ID of the namespace | +| `label` | Label | A label available on this group | | `lfsEnabled` | Boolean | Indicates if Large File Storage (LFS) is enabled for namespace | | `mentionsDisabled` | Boolean | Indicates if a group is disabled from getting mentioned | | `name` | String! | Name of the namespace | @@ -681,6 +751,20 @@ Autogenerated return type of EpicTreeReorder | `visibility` | String | Visibility of the namespace | | `webUrl` | String! | Web URL of the group | +## GroupMember + +Represents a Group Member + +| Name | Type | Description | +| --- | ---- | ---------- | +| `accessLevel` | AccessLevel | GitLab::Access level | +| `createdAt` | Time | Date and time the membership was created | +| `createdBy` | User | User that authorized membership | +| `expiresAt` | Time | Date and time the membership expires | +| `group` | Group | Group that a User is a member of | +| `updatedAt` | Time | Date and time the membership was last updated | +| `userPermissions` | GroupPermissions! | Permissions for the current user on the resource | + ## GroupPermissions | Name | Type | Description | @@ -801,9 +885,12 @@ Represents an iteration object. | Name | Type | Description | | --- | ---- | ---------- | | `createdAt` | Time | Timestamp of when the Jira import was created | +| `failedToImportCount` | Int! | Count of issues that failed to import | +| `importedIssuesCount` | Int! | Count of issues that were successfully imported | | `jiraProjectKey` | String! | Project key for the imported Jira project | | `scheduledAt` | Time | Timestamp of when the Jira import was scheduled | | `scheduledBy` | User | User that started the Jira import | +| `totalIssueCount` | Int! | Total count of issues that were attempted to import | ## JiraImportStartPayload @@ -815,13 +902,41 @@ Autogenerated return type of JiraImportStart | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `jiraImport` | JiraImport | The Jira import data after mutation | +## JiraImportUsersPayload + +Autogenerated return type of JiraImportUsers + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `jiraUsers` | JiraUser! => Array | Users returned from Jira, matched by email and name if possible. | + +## JiraProject + +| Name | Type | Description | +| --- | ---- | ---------- | +| `key` | String! | Key of the Jira project | +| `name` | String | Name of the Jira project | +| `projectId` | Int! | ID of the Jira project | + ## JiraService | Name | Type | Description | | --- | ---- | ---------- | | `active` | Boolean | Indicates if the service is active | +| `projects` | JiraProjectConnection | List of Jira projects fetched through Jira REST API | | `type` | String | Class name of the service | +## JiraUser + +| Name | Type | Description | +| --- | ---- | ---------- | +| `gitlabId` | Int | Id of the matched GitLab user | +| `jiraAccountId` | String! | Account id of the Jira user | +| `jiraDisplayName` | String! | Display name of the Jira user | +| `jiraEmail` | String | Email of the Jira user, returned only for users with public emails | + ## Label | Name | Type | Description | @@ -848,6 +963,7 @@ Autogenerated return type of MarkAsSpamSnippet | Name | Type | Description | | --- | ---- | ---------- | | `allowCollaboration` | Boolean | Indicates if members of the target project can push to the fork | +| `author` | User | User who created this merge request | | `createdAt` | Time! | Timestamp of when the merge request was created | | `defaultMergeCommitMessage` | String | Default merge commit message of the merge request | | `description` | String | Description of the merge request (Markdown rendered as HTML for caching) | @@ -868,6 +984,7 @@ Autogenerated return type of MarkAsSpamSnippet | `mergeStatus` | String | Status of the merge request | | `mergeWhenPipelineSucceeds` | Boolean | Indicates if the merge has been set to be merged when its pipeline succeeds (MWPS) | | `mergeableDiscussionsState` | Boolean | Indicates if all discussions in the merge request have been resolved, allowing the merge request to be merged | +| `mergedAt` | Time | Timestamp of when the merge request was merged, null if not merged | | `milestone` | Milestone | The milestone of the merge request | | `project` | Project! | Alias for target_project | | `projectId` | Int! | ID of the merge request project | @@ -883,6 +1000,7 @@ Autogenerated return type of MarkAsSpamSnippet | `state` | MergeRequestState! | State of the merge request | | `subscribed` | Boolean! | Indicates if the currently logged in user is subscribed to this merge request | | `targetBranch` | String! | Target branch of the merge request | +| `targetBranchExists` | Boolean! | Indicates if the target branch of the merge request exists | | `targetProject` | Project! | Target project of the merge request | | `targetProjectId` | Int! | ID of the merge request target project | | `taskCompletionStatus` | TaskCompletionStatus! | Completion status of tasks | @@ -897,6 +1015,16 @@ Autogenerated return type of MarkAsSpamSnippet | `webUrl` | String | Web URL of the merge request | | `workInProgress` | Boolean! | Indicates if the merge request is a work in progress (WIP) | +## MergeRequestCreatePayload + +Autogenerated return type of MergeRequestCreate + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `mergeRequest` | MergeRequest | The merge request after mutation | + ## MergeRequestPermissions Check permissions for the current user on a merge request @@ -984,6 +1112,7 @@ Autogenerated return type of MergeRequestSetWip | Name | Type | Description | | --- | ---- | ---------- | | `path` | String | Path to a file with the dashboard definition | +| `schemaValidationWarnings` | String! => Array | Dashboard schema validation warnings | ## MetricsDashboardAnnotation @@ -1004,9 +1133,12 @@ Represents a milestone. | `createdAt` | Time! | Timestamp of milestone creation | | `description` | String | Description of the milestone | | `dueDate` | Time | Timestamp of the milestone due date | +| `groupMilestone` | Boolean! | Indicates if milestone is at group level | | `id` | ID! | ID of the milestone | +| `projectMilestone` | Boolean! | Indicates if milestone is at project level | | `startDate` | Time | Timestamp of the milestone start date | | `state` | MilestoneStateEnum! | State of the milestone | +| `subgroupMilestone` | Boolean! | Indicates if milestone is at subgroup level | | `title` | String! | Title of the milestone | | `updatedAt` | Time! | Timestamp of last milestone update | | `webPath` | String! | Web path of the milestone | @@ -1040,9 +1172,10 @@ Represents a milestone. | `id` | ID! | ID of the note | | `position` | DiffPosition | The position of this note on a diff | | `project` | Project | Project associated with the note | -| `resolvable` | Boolean! | Indicates if this note can be resolved. That is, if it is a resolvable discussion or simply a standalone note | -| `resolvedAt` | Time | Timestamp of the note's resolution | -| `resolvedBy` | User | User that resolved the discussion | +| `resolvable` | Boolean! | Indicates if the object can be resolved | +| `resolved` | Boolean! | Indicates if the object is resolved | +| `resolvedAt` | Time | Timestamp of when the object was resolved | +| `resolvedBy` | User | User who resolved the object | | `system` | Boolean! | Indicates whether this note was created by the system or by a user | | `updatedAt` | Time! | Timestamp of the note's last activity | | `userPermissions` | NotePermissions! | Permissions for the current user on the resource | @@ -1109,6 +1242,7 @@ Information about pagination in a connection. | `finishedAt` | Time | Timestamp of the pipeline's completion | | `id` | ID! | ID of the pipeline | | `iid` | String! | Internal ID of the pipeline | +| `securityReportSummary` | SecurityReportSummary | Vulnerability and scanned resource counts for each security scanner of the pipeline | | `sha` | String! | SHA of the pipeline's commit | | `startedAt` | Time | Timestamp when the pipeline was started | | `status` | PipelineStatusEnum! | Status of the pipeline (CREATED, WAITING_FOR_RESOURCE, PREPARING, PENDING, RUNNING, FAILED, SUCCESS, CANCELED, SKIPPED, MANUAL, SCHEDULED) | @@ -1129,10 +1263,12 @@ Information about pagination in a connection. | --- | ---- | ---------- | | `alertManagementAlert` | AlertManagementAlert | A single Alert Management alert of the project | | `alertManagementAlertStatusCounts` | AlertManagementAlertStatusCountsType | Counts of alerts by status for the project | +| `allowMergeOnSkippedPipeline` | Boolean | If `only_allow_merge_if_pipeline_succeeds` is true, indicates if merge requests of the project can also be merged with skipped jobs | | `archived` | Boolean | Indicates the archived status of the project | | `autocloseReferencedIssues` | Boolean | Indicates if issues referenced by merge requests and commits within the default branch are closed automatically | | `avatarUrl` | String | URL to avatar image file of the project | | `board` | Board | A single board of the project | +| `containerExpirationPolicy` | ContainerExpirationPolicy | The container expiration policy of the project | | `containerRegistryEnabled` | Boolean | Indicates if the project stores Docker container images in a container registry | | `createdAt` | Time | Timestamp of the project creation | | `description` | String | Short description of the project | @@ -1148,6 +1284,7 @@ Information about pagination in a connection. | `issuesEnabled` | Boolean | Indicates if Issues are enabled for the current user | | `jiraImportStatus` | String | Status of Jira import background job of the project | | `jobsEnabled` | Boolean | Indicates if CI/CD pipeline jobs are enabled for the current user | +| `label` | Label | A label available on this project | | `lastActivityAt` | Time | Timestamp of the project last activity | | `lfsEnabled` | Boolean | Indicates if the project has Large File Storage (LFS) enabled | | `mergeRequest` | MergeRequest | A single merge request of the project | @@ -1160,6 +1297,7 @@ Information about pagination in a connection. | `onlyAllowMergeIfPipelineSucceeds` | Boolean | Indicates if merge requests of the project can only be merged with successful jobs | | `openIssuesCount` | Int | Number of open issues for the project | | `path` | String! | Path of the project | +| `pipeline` | Pipeline | Build pipeline of the project | | `printingMergeRequestLinkEnabled` | Boolean | Indicates if a link to create or view a merge request should display after a push to Git repositories of the project from the command line | | `publicJobs` | Boolean | Indicates if there is public access to pipelines and job details of the project, including output logs and artifacts | | `release` | Release | A single release of the project. Available only when feature flag `graphql_release_data` is enabled | @@ -1185,6 +1323,22 @@ Information about pagination in a connection. | `webUrl` | String | Web URL of the project | | `wikiEnabled` | Boolean | Indicates if Wikis are enabled for the current user | +## ProjectMember + +Represents a Project Member + +| Name | Type | Description | +| --- | ---- | ---------- | +| `accessLevel` | AccessLevel | GitLab::Access level | +| `createdAt` | Time | Date and time the membership was created | +| `createdBy` | User | User that authorized membership | +| `expiresAt` | Time | Date and time the membership expires | +| `id` | ID! | ID of the member | +| `project` | Project | Project that User is a member of | +| `updatedAt` | Time | Date and time the membership was last updated | +| `user` | User! | User that is associated with the member object | +| `userPermissions` | ProjectPermissions! | Permissions for the current user on the resource | + ## ProjectPermissions | Name | Type | Description | @@ -1248,6 +1402,7 @@ Information about pagination in a connection. | Name | Type | Description | | --- | ---- | ---------- | +| `assets` | ReleaseAssets | Assets of the release | | `author` | User | User that created the release | | `commit` | Commit | The commit associated with the release | | `createdAt` | Time | Timestamp of when the release was created | @@ -1258,6 +1413,40 @@ Information about pagination in a connection. | `tagName` | String! | Name of the tag associated with the release | | `tagPath` | String | Relative web path to the tag associated with the release | +## ReleaseAssets + +| Name | Type | Description | +| --- | ---- | ---------- | +| `assetsCount` | Int | Number of assets of the release | + +## ReleaseEvidence + +Evidence for a release + +| Name | Type | Description | +| --- | ---- | ---------- | +| `collectedAt` | Time | Timestamp when the evidence was collected | +| `filepath` | String | URL from where the evidence can be downloaded | +| `id` | ID! | ID of the evidence | +| `sha` | String | SHA1 ID of the evidence hash | + +## ReleaseLink + +| Name | Type | Description | +| --- | ---- | ---------- | +| `external` | Boolean | Indicates the link points to an external resource | +| `id` | ID! | ID of the link | +| `linkType` | ReleaseLinkType | Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other` | +| `name` | String | Name of the link | +| `url` | String | URL of the link | + +## ReleaseSource + +| Name | Type | Description | +| --- | ---- | ---------- | +| `format` | String | Format of the source | +| `url` | String | Download URL of the source | + ## RemoveAwardEmojiPayload Autogenerated return type of RemoveAwardEmoji @@ -1288,7 +1477,7 @@ Autogenerated return type of RemoveProjectFromSecurityDashboard ## Requirement -Represents a requirement. +Represents a requirement | Name | Type | Description | | --- | ---- | ---------- | @@ -1334,6 +1523,37 @@ Counts of requirements by their state. | `storageSize` | Float! | The total storage in bytes | | `wikiSize` | Float! | The wiki size in bytes | +## RunDASTScanPayload + +Autogenerated return type of RunDASTScan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `pipelineUrl` | String | URL of the pipeline that was created. | + +## SecurityReportSummary + +Represents summary of a security report + +| Name | Type | Description | +| --- | ---- | ---------- | +| `containerScanning` | SecurityReportSummarySection | Aggregated counts for the container_scanning scan | +| `dast` | SecurityReportSummarySection | Aggregated counts for the dast scan | +| `dependencyScanning` | SecurityReportSummarySection | Aggregated counts for the dependency_scanning scan | +| `sast` | SecurityReportSummarySection | Aggregated counts for the sast scan | +| `secretDetection` | SecurityReportSummarySection | Aggregated counts for the secret_detection scan | + +## SecurityReportSummarySection + +Represents a section of a summary of a security report + +| Name | Type | Description | +| --- | ---- | ---------- | +| `scannedResourcesCount` | Int | Total number of scanned resources | +| `vulnerabilitiesCount` | Int | Total number of vulnerabilities | + ## SentryDetailedError A Sentry error. @@ -1455,8 +1675,9 @@ Represents a snippet entry | Name | Type | Description | | --- | ---- | ---------- | -| `author` | User! | The owner of the snippet | +| `author` | User | The owner of the snippet | | `blob` | SnippetBlob! | Snippet blob | +| `blobs` | SnippetBlob! => Array | Snippet blobs | | `createdAt` | Time! | Timestamp this snippet was created | | `description` | String | Description of the snippet | | `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | @@ -1538,6 +1759,17 @@ Completion status of tasks | `completedCount` | Int! | Number of completed tasks | | `count` | Int! | Number of total tasks | +## TestReport + +Represents a requirement test report. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `author` | User | Author of the test report | +| `createdAt` | Time! | Timestamp of when the test report was created | +| `id` | ID! | ID of the test report | +| `state` | TestReportState! | State of the test report | + ## Timelog | Name | Type | Description | @@ -1646,6 +1878,16 @@ Autogenerated return type of UpdateAlertStatus | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `issue` | Issue | The issue created after mutation | +## UpdateContainerExpirationPolicyPayload + +Autogenerated return type of UpdateContainerExpirationPolicy + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `containerExpirationPolicy` | ContainerExpirationPolicy | The container expiration policy after mutation | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | + ## UpdateEpicPayload Autogenerated return type of UpdateEpic @@ -1713,7 +1955,7 @@ Autogenerated return type of UpdateSnippet | `avatarUrl` | String | URL of the user's avatar | | `id` | ID! | ID of the user | | `name` | String! | Human-readable name of the user | -| `state` | String! | State of the issue | +| `state` | UserState! | State of the user | | `userPermissions` | UserPermissions! | Permissions for the current user on the resource | | `username` | String! | Username of the user. Unique within this instance of GitLab | | `webUrl` | String! | Web URL of the user | @@ -1744,13 +1986,24 @@ Represents a vulnerability. | `id` | ID! | GraphQL ID of the vulnerability | | `location` | VulnerabilityLocation | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability | | `project` | Project | The project on which the vulnerability was found | -| `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST) | +| `reportType` | VulnerabilityReportType | Type of the security report that found the vulnerability (SAST, DEPENDENCY_SCANNING, CONTAINER_SCANNING, DAST, SECRET_DETECTION) | | `severity` | VulnerabilitySeverity | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL) | | `state` | VulnerabilityState | State of the vulnerability (DETECTED, DISMISSED, RESOLVED, CONFIRMED) | | `title` | String | Title of the vulnerability | +| `userNotesCount` | Int! | Number of user notes attached to the vulnerability | | `userPermissions` | VulnerabilityPermissions! | Permissions for the current user on the resource | | `vulnerabilityPath` | String | URL to the vulnerability's details page | +## VulnerabilityIssueLink + +Represents an issue link of a vulnerability. + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | GraphQL ID of the vulnerability | +| `issue` | Issue! | The issue attached to issue link | +| `linkType` | VulnerabilityIssueLinkType! | Type of the issue link | + ## VulnerabilityLocationContainerScanning Represents the location of a vulnerability found by a container security scan @@ -1793,6 +2046,18 @@ Represents the location of a vulnerability found by a SAST scan | `vulnerableClass` | String | Class containing the vulnerability | | `vulnerableMethod` | String | Method containing the vulnerability | +## VulnerabilityLocationSecretDetection + +Represents the location of a vulnerability found by a secret detection scan + +| Name | Type | Description | +| --- | ---- | ---------- | +| `endLine` | String | Number of the last relevant line in the vulnerable file | +| `file` | String | Path to the vulnerable file | +| `startLine` | String | Number of the first relevant line in the vulnerable file | +| `vulnerableClass` | String | Class containing the vulnerability | +| `vulnerableMethod` | String | Method containing the vulnerability | + ## VulnerabilityPermissions Check permissions for the current user on a vulnerability diff --git a/doc/api/group_activity_analytics.md b/doc/api/group_activity_analytics.md index a211097e30b..302c7703669 100644 --- a/doc/api/group_activity_analytics.md +++ b/doc/api/group_activity_analytics.md @@ -17,7 +17,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/analytics/group_activity/issues_count?group_path=gitlab-org +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/analytics/group_activity/issues_count?group_path=gitlab-org" ``` Example response: @@ -41,7 +41,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/analytics/group_activity/merge_requests_count?group_path=gitlab-org +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/analytics/group_activity/merge_requests_count?group_path=gitlab-org" ``` Example response: @@ -65,7 +65,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/analytics/group_activity/new_members_count?group_path=gitlab-org +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/analytics/group_activity/new_members_count?group_path=gitlab-org" ``` Example response: diff --git a/doc/api/group_badges.md b/doc/api/group_badges.md index 5b7164cdd4d..43e1944226d 100644 --- a/doc/api/group_badges.md +++ b/doc/api/group_badges.md @@ -28,7 +28,7 @@ GET /groups/:id/badges | `name` | string | no | Name of the badges to return (case-sensitive). | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/badges?name=Coverage +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/badges?name=Coverage" ``` Example response: @@ -61,7 +61,7 @@ GET /groups/:id/badges/:badge_id | `badge_id` | integer | yes | The badge ID | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id" ``` Example response: @@ -92,7 +92,7 @@ POST /groups/:id/badges | `image_url` | string | yes | URL of the badge image | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&position=0" https://gitlab.example.com/api/v4/groups/:id/badges +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&position=0" "https://gitlab.example.com/api/v4/groups/:id/badges" ``` Example response: @@ -124,7 +124,7 @@ PUT /groups/:id/badges/:badge_id | `image_url` | string | no | URL of the badge image | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id" ``` Example response: @@ -154,7 +154,7 @@ DELETE /groups/:id/badges/:badge_id | `badge_id` | integer | yes | The badge ID | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id" ``` ## Preview a badge from a group @@ -172,7 +172,7 @@ GET /groups/:id/badges/render | `image_url` | string | yes | URL of the badge image | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/badges/render?link_url=http%3A%2F%2Fexample.com%2Fci_status.svg%3Fproject%3D%25%7Bproject_path%7D%26ref%3D%25%7Bdefault_branch%7D&image_url=https%3A%2F%2Fshields.io%2Fmy%2Fbadge +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/badges/render?link_url=http%3A%2F%2Fexample.com%2Fci_status.svg%3Fproject%3D%25%7Bproject_path%7D%26ref%3D%25%7Bdefault_branch%7D&image_url=https%3A%2F%2Fshields.io%2Fmy%2Fbadge" ``` Example response: diff --git a/doc/api/group_boards.md b/doc/api/group_boards.md index adfcd6e65cb..4ff373ce583 100644 --- a/doc/api/group_boards.md +++ b/doc/api/group_boards.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Group Issue Boards API Every API call to group boards must be authenticated. @@ -18,7 +24,7 @@ GET /groups/:id/boards | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards" ``` Example response: @@ -136,7 +142,7 @@ GET /groups/:id/boards/:board_id | `board_id` | integer | yes | The ID of a board | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards/1" ``` Example response: @@ -250,7 +256,7 @@ POST /groups/:id/boards | `name` | string | yes | The name of the new board | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards?name=newboard +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards?name=newboard" ``` Example response: @@ -321,7 +327,7 @@ PUT /groups/:id/boards/:board_id | `weight` | integer | no | The weight range from 0 to 9, to which the board should be scoped to | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards/1?name=new_name&milestone_id=44&assignee_id=1&labels=GroupLabel&weight=4 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards/1?name=new_name&milestone_id=44&assignee_id=1&labels=GroupLabel&weight=4" ``` Example response: @@ -382,7 +388,7 @@ DELETE /groups/:id/boards/:board_id | `board_id` | integer | yes | The ID of a board | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards/1" ``` ## List group issue board lists @@ -400,7 +406,7 @@ GET /groups/:id/boards/:board_id/lists | `board_id` | integer | yes | The ID of a board | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards/1/lists +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards/1/lists" ``` Example response: @@ -452,7 +458,7 @@ GET /groups/:id/boards/:board_id/lists/:list_id | `list_id` | integer | yes | The ID of a board's list | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards/1/lists/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards/1/lists/1" ``` Example response: @@ -484,7 +490,7 @@ POST /groups/:id/boards/:board_id/lists | `label_id` | integer | yes | The ID of a label | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4/boards/12/lists?milestone_id=7 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/4/boards/12/lists?milestone_id=7" ``` Example response: @@ -526,7 +532,7 @@ PUT /groups/:id/boards/:board_id/lists/:list_id | `position` | integer | yes | The position of the list | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/group/5/boards/1/lists/1?position=2 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/group/5/boards/1/lists/1?position=2" ``` Example response: @@ -558,5 +564,5 @@ DELETE /groups/:id/boards/:board_id/lists/:list_id | `list_id` | integer | yes | The ID of a board's list | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/boards/1/lists/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/boards/1/lists/1" ``` diff --git a/doc/api/group_clusters.md b/doc/api/group_clusters.md index 01c6d59f60d..b600a92bc38 100644 --- a/doc/api/group_clusters.md +++ b/doc/api/group_clusters.md @@ -22,7 +22,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters +curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/groups/26/clusters" ``` Example response: @@ -90,7 +90,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/18 +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/groups/26/clusters/18" ``` Example response: @@ -166,7 +166,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/user \ +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/groups/26/clusters/user" \ -H "Accept: application/json" \ -H "Content-Type:application/json" \ --request POST --data '{"name":"cluster-5", "platform_kubernetes_attributes":{"api_url":"https://35.111.51.20","token":"12345","ca_cert":"-----BEGIN CERTIFICATE-----\r\nhFiK1L61owwDQYJKoZIhvcNAQELBQAw\r\nLzEtMCsGA1UEAxMkZDA1YzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM4ZDBj\r\nMB4XDTE4MTIyNzIwMDM1MVoXDTIzMTIyNjIxMDM1MVowLzEtMCsGA1UEAxMkZDA1\r\nYzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM.......-----END CERTIFICATE-----"}}' @@ -238,7 +238,7 @@ through the ["Add existing cluster to group"](#add-existing-cluster-to-group) en Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/24 \ +curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/groups/26/clusters/24" \ -H "Content-Type:application/json" \ --request PUT --data '{"name":"new-cluster-name","domain":"new-domain.com","api_url":"https://new-api-url.com"}' ``` @@ -307,5 +307,5 @@ Parameters: Example request: ```shell -curl --request DELETE --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/23 +curl --request DELETE --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/groups/26/clusters/23" ``` diff --git a/doc/api/group_import_export.md b/doc/api/group_import_export.md index 355ecbfb98f..01d81eb62ac 100644 --- a/doc/api/group_import_export.md +++ b/doc/api/group_import_export.md @@ -28,7 +28,7 @@ POST /groups/:id/export | `id` | integer/string | yes | ID of the group owned by the authenticated user | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/export +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/export" ``` ```json @@ -50,7 +50,7 @@ GET /groups/:id/export/download | `id` | integer/string | yes | ID of the group owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" --remote-header-name --remote-name https://gitlab.example.com/api/v4/groups/1/export/download +curl --header "PRIVATE-TOKEN: <your_access_token>" --remote-header-name --remote-name "https://gitlab.example.com/api/v4/groups/1/export/download" ``` ```shell @@ -83,9 +83,13 @@ The `file=` parameter must point to a file on your file system and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "name=imported-group" --form "path=imported-group" --form "file=@/path/to/file" https://gitlab.example.com/api/v4/groups/import +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "name=imported-group" --form "path=imported-group" --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/groups/import" ``` +NOTE: **Note:** +The maximum import file size can be set by the Administrator, default is 50MB. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin UI](../user/admin_area/settings/account_and_limit_settings.md). + ## Important notes Note the following: diff --git a/doc/api/group_labels.md b/doc/api/group_labels.md index c3b86233836..5ae5ea4286a 100644 --- a/doc/api/group_labels.md +++ b/doc/api/group_labels.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Group Labels API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21368) in GitLab 11.8. @@ -22,7 +28,7 @@ GET /groups/:id/labels | `include_ancestor_groups` | boolean | no | Include ancestor groups. Defaults to `true`. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/labels?with_counts=true +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/labels?with_counts=true" ``` Example response: @@ -71,7 +77,7 @@ GET /groups/:id/labels/:label_id | `include_ancestor_groups` | boolean | no | Include ancestor groups. Defaults to `true`. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/labels/bug +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/labels/bug" ``` Example response: @@ -107,7 +113,7 @@ POST /groups/:id/labels | `description` | string | no | The description of the label, | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data '{"name": "Feature Proposal", "color": "#FFA500", "description": "Describes new ideas" }' https://gitlab.example.com/api/v4/groups/5/labels +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data '{"name": "Feature Proposal", "color": "#FFA500", "description": "Describes new ideas" }' "https://gitlab.example.com/api/v4/groups/5/labels" ``` Example response: @@ -144,7 +150,7 @@ PUT /groups/:id/labels/:label_id | `description` | string | no | The description of the label. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data '{"new_name": "Feature Idea" }' https://gitlab.example.com/api/v4/groups/5/labels/Feature%20Proposal +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" --data '{"new_name": "Feature Idea" }' "https://gitlab.example.com/api/v4/groups/5/labels/Feature%20Proposal" ``` Example response: @@ -180,7 +186,7 @@ DELETE /groups/:id/labels/:label_id | `label_id` | integer or string | yes | The ID or title of a group's label. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/labels/bug +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/labels/bug" ``` NOTE: **Note:** An older endpoint `DELETE /groups/:id/labels` with `name` in the parameters is still available, but deprecated. @@ -200,7 +206,7 @@ POST /groups/:id/labels/:label_id/subscribe | `label_id` | integer or string | yes | The ID or title of a group's label. | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/labels/9/subscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/labels/9/subscribe" ``` Example response: @@ -236,7 +242,7 @@ POST /groups/:id/labels/:label_id/unsubscribe | `label_id` | integer or string | yes | The ID or title of a group's label. | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/labels/9/unsubscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/labels/9/unsubscribe" ``` Example response: diff --git a/doc/api/group_level_variables.md b/doc/api/group_level_variables.md index e30ee329114..aa5f0b3db72 100644 --- a/doc/api/group_level_variables.md +++ b/doc/api/group_level_variables.md @@ -1,6 +1,6 @@ # Group-level Variables API -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/34519) in GitLab 9.5 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34519) in GitLab 9.5 ## List group variables diff --git a/doc/api/group_milestones.md b/doc/api/group_milestones.md index 10445acf881..e157655a713 100644 --- a/doc/api/group_milestones.md +++ b/doc/api/group_milestones.md @@ -1,7 +1,16 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Group milestones API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12819) in GitLab 9.5. +This page describes the group milestones API. +There's a separate [project milestones API](./group_milestones.md) page. + ## List group milestones Returns a list of group milestones. @@ -27,7 +36,7 @@ Parameters: | `search` | string | no | Return only milestones with a title or description matching the provided string | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/milestones +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/milestones" ``` Example Response: @@ -150,7 +159,7 @@ Parameters: ## Get all burndown chart events for a single milestone **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4737) in GitLab 12.1 +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4737) in GitLab 12.1 Get all burndown chart events for a single milestone. diff --git a/doc/api/groups.md b/doc/api/groups.md index bc7bff2964b..e58506380d1 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -20,6 +20,7 @@ Parameters: | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | | `owned` | boolean | no | Limit to groups explicitly owned by the current user | | `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md) | +| `top_level_only` | boolean | no | Limit to top level groups, excluding all subgroups | ```plaintext GET /groups @@ -399,13 +400,13 @@ The `projects` and `shared_projects` attributes in the response are deprecated a To get the details of all projects within a group, use either the [list a group's projects](#list-a-groups-projects) or the [list a group's shared projects](#list-a-groups-shared-projects) endpoint. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/4" ``` This endpoint returns: - All projects and shared projects in GitLab 12.5 and earlier. -- A maximum of 100 projects and shared projects [in GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/31031) +- A maximum of 100 projects and shared projects [in GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/31031) and later. To get the details of all projects within a group, use the [list a group's projects endpoint](#list-a-groups-projects) instead. @@ -427,6 +428,15 @@ Example response: "file_template_project_id": 1, "parent_id": null, "created_at": "2020-01-15T12:36:29.590Z", + "shared_with_groups": [ + { + "group_id": 28, + "group_name": "H5bp", + "group_full_path": "h5bp", + "group_access_level": 20, + "expires_at": null + } + ], "projects": [ // Deprecated and will be removed in API v5 { "id": 7, @@ -577,10 +587,22 @@ Additional response parameters: } ``` +Users on GitLab [Silver, Premium, or higher](https://about.gitlab.com/pricing/) will also see +the `marked_for_deletion_on` attribute: + +```json +{ + "id": 4, + "description": "Aliquid qui quis dignissimos distinctio ut commodi voluptas est.", + "marked_for_deletion_on": "2020-04-03", + ... +} +``` + When adding the parameter `with_projects=false`, projects will not be returned. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4?with_projects=false +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/4?with_projects=false" ``` Example response: @@ -638,7 +660,7 @@ Parameters: | `auto_devops_enabled` | boolean | no | Default to Auto DevOps pipeline for all projects within this group. | | `subgroup_creation_level` | string | no | Allowed to create subgroups. Can be `owner` (Owners), or `maintainer` (Maintainers). | | `emails_disabled` | boolean | no | Disable email notifications | -| `avatar` | mixed | no | Image file for avatar of the group. [Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/36681) | +| `avatar` | mixed | no | Image file for avatar of the group. [Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/36681) | | `mentions_disabled` | boolean | no | Disable the capability of a group from getting mentioned | | `lfs_enabled` | boolean | no | Enable/disable Large File Storage (LFS) for the projects in this group. | | `request_access_enabled` | boolean | no | Allow users to request member access. | @@ -657,6 +679,19 @@ The `default_branch_protection` attribute determines whether developers and main | `1` | Partial protection. Developers and maintainers can: <br>- Push new commits | | `2` | Full protection. Only maintainers can: <br>- Push new commits | +## New Subgroup + +This is similar to creating a [New group](#new-group). You'll need the `parent_id` from the [List groups](#list-groups) call. You can then enter the desired: + +- `subgroup_path` +- `subgroup_name` + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --header "Content-Type: application/json" \ + --data '{"path": "<subgroup_path>", "name": "<subgroup_name>", "parent_id": <parent_group_id> } \ + "https://gitlab.example.com/api/v4/groups/" +``` + ## Transfer project to group Transfer a project to the Group namespace. Available only to instance administrators, although an [alternative API endpoint](projects.md#transfer-a-project-to-a-new-namespace) is available which does not require instance administrator access. Transferring projects may fail when tagged packages exist in the project's repository. @@ -673,7 +708,7 @@ Parameters: | `project_id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4/projects/56 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/4/projects/56" ``` ## Update group @@ -699,7 +734,7 @@ PUT /groups/:id | `auto_devops_enabled` | boolean | no | Default to Auto DevOps pipeline for all projects within this group. | | `subgroup_creation_level` | string | no | Allowed to create subgroups. Can be `owner` (Owners), or `maintainer` (Maintainers). | | `emails_disabled` | boolean | no | Disable email notifications | -| `avatar` | mixed | no | Image file for avatar of the group. [Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/36681) | +| `avatar` | mixed | no | Image file for avatar of the group. [Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/36681) | | `mentions_disabled` | boolean | no | Disable the capability of a group from getting mentioned | | `lfs_enabled` (optional) | boolean | no | Enable/disable Large File Storage (LFS) for the projects in this group. | | `request_access_enabled` | boolean | no | Allow users to request member access. | @@ -719,7 +754,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab This endpoint returns: - All projects and shared projects in GitLab 12.5 and earlier. -- A maximum of 100 projects and shared projects [in GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/31031) +- A maximum of 100 projects and shared projects [in GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/31031) and later. To get the details of all projects within a group, use the [list a group's projects endpoint](#list-a-groups-projects) instead. @@ -803,7 +838,7 @@ Only available to group owners and administrators. This endpoint either: - Removes group, and queues a background job to delete all projects in the group as well. -- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). ```plaintext DELETE /groups/:id @@ -819,7 +854,7 @@ The response will be `202 Accepted` if the user has authorization. ## Restore group marked for deletion **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/33257) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33257) in GitLab 12.8. Restores a group marked for deletion. @@ -1088,3 +1123,35 @@ Read more in the [Group Badges](group_badges.md) documentation. ## Group Import/Export Read more in the [Group Import/Export](group_import_export.md) documentation. + +## Share Groups with Groups + +These endpoints create and delete links for sharing a group with another group. For more information, see the related discussion in the [GitLab Groups](../user/group/index.md#sharing-a-group-with-another-group) page. + +### Create a link to share a group with another group + +Share group with another group. Returns `200` and the [group details](#details-of-a-group) on success. + +```plaintext +POST /groups/:id/share +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `group_id` | integer | yes | The ID of the group to share with | +| `group_access` | integer | yes | The [permissions level](members.md) to grant the group | +| `expires_at` | string | no | Share expiration date in ISO 8601 format: 2016-09-26 | + +### Delete link sharing group with another group + +Unshare the group from another group. Returns `204` and no content on success. + +```plaintext +DELETE /groups/:id/share/:group_id +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `group_id` | integer | yes | The ID of the group to share with | diff --git a/doc/api/import.md b/doc/api/import.md index 9640ba19cf9..307796f8acb 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -16,7 +16,7 @@ POST /import/github | `target_namespace` | string | yes | Namespace to import repository into | ```shell -curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --data "personal_access_token=abc123&repo_id=12345&target_namespace=root" https://gitlab.example.com/api/v4/import/github +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "personal_access_token=abc123&repo_id=12345&target_namespace=root" "https://gitlab.example.com/api/v4/import/github" ``` Example response: diff --git a/doc/api/instance_level_ci_variables.md b/doc/api/instance_level_ci_variables.md index d0871fdf4a7..72d20109fbd 100644 --- a/doc/api/instance_level_ci_variables.md +++ b/doc/api/instance_level_ci_variables.md @@ -67,8 +67,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a Create a new instance-level variable. -NOTE: **Note:** -The maximum number of instance-level variables is [planned to be 25](https://gitlab.com/gitlab-org/gitlab/-/issues/216097). +[Since GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/216097), the maximum number of allowed instance-level variables can be changed. ```plaintext POST /admin/ci/variables @@ -77,7 +76,7 @@ POST /admin/ci/variables | Attribute | Type | required | Description | |-----------------|---------|----------|-----------------------| | `key` | string | yes | The `key` of a variable. Max 255 characters, only `A-Z`, `a-z`, `0-9`, and `_` are allowed. | -| `value` | string | yes | The `value` of a variable. | +| `value` | string | yes | The `value` of a variable. Around 700 characters allowed. | | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file`. | | `protected` | boolean | no | Whether the variable is protected. | | `masked` | boolean | no | Whether the variable is masked. | diff --git a/doc/api/issues.md b/doc/api/issues.md index 8e5882c4d4e..f640300e3ae 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Issues API Every API call to issues must be authenticated. @@ -70,7 +76,7 @@ GET /issues?confidential=true | `non_archived` | boolean | no | Return issues only from non-archived projects. If `false`, response will return issues from both archived and non-archived projects. Default is `true`. _(Introduced in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/197170))_ | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/issues +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/issues" ``` Example response: @@ -226,7 +232,7 @@ GET /groups/:id/issues?confidential=true | `non_archived` | boolean | no | Return issues from non archived projects. Default is true. _(Introduced in [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23785))_ | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4/issues +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/4/issues" ``` Example response: @@ -357,7 +363,7 @@ GET /projects/:id/issues?confidential=true | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `iids[]` | integer array | no | Return only the milestone having the given `iid` | +| `iids[]` | integer array | no | Return only the issues having the given `iid` | | `state` | string | no | Return all issues or just those that are `opened` or `closed` | | `labels` | string | no | Comma-separated list of label names, issues must have all labels to be returned. `None` lists all issues with no labels. `Any` lists all issues with at least one label. `No+Label` (Deprecated) lists all issues with no labels. Predefined names are case-insensitive. | | `with_labels_details` | boolean | no | If `true`, response will return more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. `description_html` Introduced in [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21413) | @@ -380,7 +386,7 @@ GET /projects/:id/issues?confidential=true | `not` | Hash | no | Return issues that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `my_reaction_emoji`, `search`, `in` | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues" ``` Example response: @@ -508,7 +514,7 @@ GET /projects/:id/issues/:issue_iid | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues/41 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/41" ``` Example response: @@ -631,7 +637,7 @@ the `epic` property: **Note**: The `closed_by` attribute was [introduced in GitLab 10.6](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17042). This value will only be present for issues which were closed after GitLab 10.6 and when the user account that closed the issue still exists. -**Note**: The `epic_iid` attribute is deprecated and [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/issues/35157). +**Note**: The `epic_iid` attribute is deprecated and [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157). Please use `iid` of the `epic` attribute instead. ## New issue @@ -658,10 +664,10 @@ POST /projects/:id/issues | `discussion_to_resolve` | string | no | The ID of a discussion to resolve. This will fill in the issue with a default description and mark the discussion as resolved. Use in combination with `merge_request_to_resolve_discussions_of`. | | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. | | `epic_id` **(ULTIMATE)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/issues/35157)) | +| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues?title=Issues%20with%20auth&labels=bug +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues?title=Issues%20with%20auth&labels=bug" ``` Example response: @@ -768,16 +774,18 @@ PUT /projects/:id/issues/:issue_iid | `assignee_ids` | integer array | no | The ID of the user(s) to assign the issue to. Set to `0` or provide an empty value to unassign all assignees. | | `milestone_id` | integer | no | The global ID of a milestone to assign the issue to. Set to `0` or provide an empty value to unassign a milestone.| | `labels` | string | no | Comma-separated label names for an issue. Set to an empty string to unassign all labels. | +| `add_labels` | string | no | Comma-separated label names to add to an issue. | +| `remove_labels`| string | no | Comma-separated label names to remove from an issue. | | `state_event` | string | no | The state event of an issue. Set `close` to close the issue and `reopen` to reopen it | | `updated_at` | string | no | Date time string, ISO 8601 formatted, for example `2016-03-11T03:45:40Z` (requires admin or project owner rights). Empty string or null values are not accepted.| | `due_date` | string | no | Date time string in the format YEAR-MONTH-DAY, for example `2016-03-11` | | `weight` **(STARTER)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. 0 | | `discussion_locked` | boolean | no | Flag indicating if the issue's discussion is locked. If the discussion is locked only project members can add or edit comments. | | `epic_id` **(ULTIMATE)** | integer | no | ID of the epic to add the issue to. Valid values are greater than or equal to 0. | -| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/issues/35157)) | +| `epic_iid` **(ULTIMATE)** | integer | no | IID of the epic to add the issue to. Valid values are greater than or equal to 0. (deprecated, [will be removed in version 5](https://gitlab.com/gitlab-org/gitlab/-/issues/35157)) | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues/85?state_event=close +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/85?state_event=close" ``` Example response: @@ -883,7 +891,7 @@ DELETE /projects/:id/issues/:issue_iid | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues/85 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/85" ``` ## Move an issue @@ -906,7 +914,7 @@ POST /projects/:id/issues/:issue_iid/move | `to_project_id` | integer | yes | The ID of the new project | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" --form to_project_id=5 https://gitlab.example.com/api/v4/projects/4/issues/85/move +curl --header "PRIVATE-TOKEN: <your_access_token>" --form to_project_id=5 "https://gitlab.example.com/api/v4/projects/4/issues/85/move" ``` Example response: @@ -1012,7 +1020,7 @@ POST /projects/:id/issues/:issue_iid/subscribe | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/subscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/subscribe" ``` Example response: @@ -1118,7 +1126,7 @@ POST /projects/:id/issues/:issue_iid/unsubscribe | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/unsubscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/unsubscribe" ``` Example response: @@ -1189,7 +1197,7 @@ POST /projects/:id/issues/:issue_iid/todo | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/todo +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/todo" ``` Example response: @@ -1304,7 +1312,7 @@ POST /projects/:id/issues/:issue_iid/time_estimate | `duration` | string | yes | The duration in human format. e.g: 3h30m | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/time_estimate?duration=3h30m +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/time_estimate?duration=3h30m" ``` Example response: @@ -1332,7 +1340,7 @@ POST /projects/:id/issues/:issue_iid/reset_time_estimate | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/reset_time_estimate +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/reset_time_estimate" ``` Example response: @@ -1361,7 +1369,7 @@ POST /projects/:id/issues/:issue_iid/add_spent_time | `duration` | string | yes | The duration in human format. e.g: 3h30m | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/add_spent_time?duration=1h +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/add_spent_time?duration=1h" ``` Example response: @@ -1389,7 +1397,7 @@ POST /projects/:id/issues/:issue_iid/reset_spent_time | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/reset_spent_time +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/reset_spent_time" ``` Example response: @@ -1415,7 +1423,7 @@ GET /projects/:id/issues/:issue_iid/time_stats | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/time_stats +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/time_stats" ``` Example response: @@ -1443,7 +1451,7 @@ GET /projects/:id/issues/:issue_id/related_merge_requests | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/11/related_merge_requests +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/11/related_merge_requests" ``` Example response: @@ -1599,7 +1607,7 @@ GET /projects/:id/issues/:issue_iid/closed_by | `issue_iid` | integer | yes | The internal ID of a project issue | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/issues/11/closed_by +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/issues/11/closed_by" ``` Example response: @@ -1672,7 +1680,7 @@ GET /projects/:id/issues/:issue_iid/participants | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/participants +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/participants" ``` Example response: @@ -1716,7 +1724,7 @@ GET /projects/:id/issues/:issue_iid/user_agent_detail | `issue_iid` | integer | yes | The internal ID of a project's issue | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/93/user_agent_detail +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/93/user_agent_detail" ``` Example response: diff --git a/doc/api/issues_statistics.md b/doc/api/issues_statistics.md index 8db99e93f79..8e2dcc07af8 100644 --- a/doc/api/issues_statistics.md +++ b/doc/api/issues_statistics.md @@ -46,7 +46,7 @@ GET /issues_statistics?confidential=true | `confidential` | boolean | no | Filter confidential or public issues. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/issues_statistics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/issues_statistics" ``` Example response: @@ -102,7 +102,7 @@ GET /groups/:id/issues_statistics?confidential=true | `confidential` | boolean | no | Filter confidential or public issues. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4/issues_statistics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/4/issues_statistics" ``` Example response: @@ -158,7 +158,7 @@ GET /projects/:id/issues_statistics?confidential=true | `confidential` | boolean | no | Filter confidential or public issues. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/issues_statistics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues_statistics" ``` Example response: diff --git a/doc/api/jobs.md b/doc/api/jobs.md index c06dc56407c..4dc29fc897d 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -14,7 +14,7 @@ GET /projects/:id/jobs | `scope` | string **or** array of strings | no | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. | ```shell -curl --globoff --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/1/jobs?scope[]=pending&scope[]=running' +curl --globoff --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs?scope[]=pending&scope[]=running" ``` Example of response @@ -149,7 +149,7 @@ GET /projects/:id/pipelines/:pipeline_id/jobs | `scope` | string **or** array of strings | no | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/1/pipelines/6/jobs?scope[]=pending&scope[]=running' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipelines/6/jobs?scope[]=pending&scope[]=running" ``` Example of response @@ -269,6 +269,90 @@ Example of response ] ``` +## List pipeline bridges + +Get a list of bridge jobs for a pipeline. + +```plaintext +GET /projects/:id/pipelines/:pipeline_id/bridges +``` + +| Attribute | Type | Required | Description | +|---------------|--------------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `pipeline_id` | integer | yes | ID of a pipeline. | +| `scope` | string **or** array of strings | no | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/1/pipelines/6/bridges?scope[]=pending&scope[]=running' +``` + +Example of response + +```json +[ + { + "commit": { + "author_email": "admin@example.com", + "author_name": "Administrator", + "created_at": "2015-12-24T16:51:14.000+01:00", + "id": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "message": "Test the CI integration.", + "short_id": "0ff3ae19", + "title": "Test the CI integration." + }, + "coverage": null, + "allow_failure": false, + "created_at": "2015-12-24T15:51:21.802Z", + "started_at": "2015-12-24T17:54:27.722Z", + "finished_at": "2015-12-24T17:58:27.895Z", + "duration": 240, + "id": 7, + "name": "teaspoon", + "pipeline": { + "id": 6, + "ref": "master", + "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", + "status": "pending", + "created_at": "2015-12-24T15:50:16.123Z", + "updated_at": "2015-12-24T18:00:44.432Z", + "web_url": "https://example.com/foo/bar/pipelines/6" + }, + "ref": "master", + "stage": "test", + "status": "pending", + "tag": false, + "web_url": "https://example.com/foo/bar/-/jobs/7", + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.dev/root", + "created_at": "2015-12-21T13:14:24.077Z", + "bio": null, + "location": null, + "public_email": "", + "skype": "", + "linkedin": "", + "twitter": "", + "website_url": "", + "organization": "" + }, + "downstream_pipeline": { + "id": 5, + "sha": "f62a4b2fb89754372a346f24659212eb8da13601", + "ref": "master", + "status": "pending", + "created_at": "2015-12-24T17:54:27.722Z", + "updated_at": "2015-12-24T17:58:27.896Z", + "web_url": "https://example.com/diaspora/diaspora-client/pipelines/5" + } + } +] +``` + ## Get a single job Get a single job of a project diff --git a/doc/api/keys.md b/doc/api/keys.md index 81ebd70be52..6294ac300ce 100644 --- a/doc/api/keys.md +++ b/doc/api/keys.md @@ -15,7 +15,7 @@ GET /keys/:id Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/keys/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/keys/1" ``` ```json @@ -74,7 +74,7 @@ GET /keys Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/keys?fingerprint=ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/keys?fingerprint=ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1" ``` If using sha256 fingerprint API calls, make sure that the fingerprint is URL-encoded. @@ -82,7 +82,7 @@ If using sha256 fingerprint API calls, make sure that the fingerprint is URL-enc For example, `/` is represented by `%2F` and `:` is represented by`%3A`: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/keys?fingerprint=SHA256%3AnUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo%2FlCg +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/keys?fingerprint=SHA256%3AnUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo%2FlCg" ``` Example response: @@ -133,7 +133,7 @@ Example response: ## Get user by deploy key fingerprint -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/119209) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119209) in GitLab 12.7. Deploy keys are bound to the creating user, so if you query with a deploy key fingerprint you get additional information about the projects using that key. @@ -141,7 +141,7 @@ fingerprint you get additional information about the projects using that key. Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/keys?fingerprint=SHA256%3AnUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo%2FlCg +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/keys?fingerprint=SHA256%3AnUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo%2FlCg" ``` Example response: diff --git a/doc/api/labels.md b/doc/api/labels.md index 3ced7da8ed5..3ab059fca7c 100644 --- a/doc/api/labels.md +++ b/doc/api/labels.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Labels API NOTE: **Note:** @@ -20,7 +26,7 @@ GET /projects/:id/labels | `include_ancestor_groups` | boolean | no | Include ancestor groups. Defaults to `true`. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/labels?with_counts=true +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/labels?with_counts=true" ``` Example response: @@ -115,7 +121,7 @@ GET /projects/:id/labels/:label_id | `include_ancestor_groups` | boolean | no | Include ancestor groups. Defaults to `true`. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/labels/bug +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/labels/bug" ``` Example response: @@ -291,7 +297,7 @@ POST /projects/:id/labels/:label_id/subscribe | `label_id` | integer or string | yes | The ID or title of a project's label | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/labels/1/subscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/labels/1/subscribe" ``` Example response: @@ -329,5 +335,5 @@ POST /projects/:id/labels/:label_id/unsubscribe | `label_id` | integer or string | yes | The ID or title of a project's label | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/labels/1/unsubscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/labels/1/unsubscribe" ``` diff --git a/doc/api/lint.md b/doc/api/lint.md index f0e4ad5655a..b5889884e48 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -13,7 +13,7 @@ POST /ci/lint | `content` | string | yes | the `.gitlab-ci.yaml` content| ```shell -curl --header "Content-Type: application/json" https://gitlab.example.com/api/v4/ci/lint --data '{"content": "{ \"image\": \"ruby:2.6\", \"services\": [\"postgres\"], \"before_script\": [\"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}' +curl --header "Content-Type: application/json" "https://gitlab.example.com/api/v4/ci/lint" --data '{"content": "{ \"image\": \"ruby:2.6\", \"services\": [\"postgres\"], \"before_script\": [\"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}' ``` Be sure to copy paste the exact contents of `.gitlab-ci.yml` as YAML is very picky about indentation and spaces. diff --git a/doc/api/managed_licenses.md b/doc/api/managed_licenses.md index 13eb3a3fea7..984cfa92d3a 100644 --- a/doc/api/managed_licenses.md +++ b/doc/api/managed_licenses.md @@ -13,7 +13,7 @@ GET /projects/:id/managed_licenses | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/managed_licenses +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/managed_licenses" ``` Example response: diff --git a/doc/api/markdown.md b/doc/api/markdown.md index 45f105b4e2a..32810ee349e 100644 --- a/doc/api/markdown.md +++ b/doc/api/markdown.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Markdown API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18926) in GitLab 11.0. @@ -17,7 +23,7 @@ POST /api/v4/markdown | `project` | string | no (optional) | Use `project` as a context when creating references using GitLab Flavored Markdown. [Authentication](README.md#authentication) is required if a project is not public. | ```shell -curl --header Content-Type:application/json --data '{"text":"Hello world! :tada:", "gfm":true, "project":"group_example/project_example"}' https://gitlab.example.com/api/v4/markdown +curl --header Content-Type:application/json --data '{"text":"Hello world! :tada:", "gfm":true, "project":"group_example/project_example"}' "https://gitlab.example.com/api/v4/markdown" ``` Response example: diff --git a/doc/api/members.md b/doc/api/members.md index afeda7780d7..dadd609b7ed 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -4,13 +4,17 @@ The access levels are defined in the `Gitlab::Access` module. Currently, these levels are recognized: -```plaintext -10 => Guest access -20 => Reporter access -30 => Developer access -40 => Maintainer access -50 => Owner access # Only valid for groups -``` +- No access (`0`) +- Guest (`10`) +- Reporter (`20`) +- Developer (`30`) +- Maintainer (`40`) +- Owner (`50`) - Only valid to set for groups + +CAUTION: **Caution:** +Due to [an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219299), +projects in personal namespaces will not show owner (`50`) permission +for owner. ## List all members of a group or project @@ -31,8 +35,8 @@ GET /projects/:id/members | `user_ids` | array of integers | no | Filter the results on the given user IDs | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/members -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/members +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/members" ``` Example response: @@ -59,6 +63,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "expires_at": "2012-10-22T14:13:35Z", "access_level": 30, + "email": "john@example.com", "group_saml_identity": { "extern_uid":"ABC-1234567890", "provider": "group_saml", @@ -88,8 +93,8 @@ GET /projects/:id/members/all | `user_ids` | array of integers | no | Filter the results on the given user IDs | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/members/all -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/members/all +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members/all" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/members/all" ``` Example response: @@ -116,6 +121,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "expires_at": "2012-10-22T14:13:35Z", "access_level": 30 + "email": "john@example.com", "group_saml_identity": { "extern_uid":"ABC-1234567890", "provider": "group_saml", @@ -151,8 +157,8 @@ GET /projects/:id/members/:user_id | `user_id` | integer | yes | The user ID of the member | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/members/:user_id -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/members/:user_id +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members/:user_id" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/members/:user_id" ``` Example response: @@ -188,8 +194,8 @@ GET /projects/:id/members/all/:user_id | `user_id` | integer | yes | The user ID of the member | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/members/all/:user_id -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/members/all/:user_id +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members/all/:user_id" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/members/all/:user_id" ``` Example response: @@ -225,8 +231,8 @@ POST /projects/:id/members | `expires_at` | string | no | A date string in the format YEAR-MONTH-DAY | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "user_id=1&access_level=30" https://gitlab.example.com/api/v4/groups/:id/members -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "user_id=1&access_level=30" https://gitlab.example.com/api/v4/projects/:id/members +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "user_id=1&access_level=30" "https://gitlab.example.com/api/v4/groups/:id/members" +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "user_id=1&access_level=30" "https://gitlab.example.com/api/v4/projects/:id/members" ``` Example response: @@ -262,8 +268,8 @@ PUT /projects/:id/members/:user_id | `expires_at` | string | no | A date string in the format YEAR-MONTH-DAY | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/members/:user_id?access_level=40 -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/members/:user_id?access_level=40 +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members/:user_id?access_level=40" +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/members/:user_id?access_level=40" ``` Example response: @@ -284,7 +290,7 @@ Example response: ### Set override flag for a member of a group -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4875) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4875) in GitLab 12.10. By default, the access level of LDAP group members is set to the value specified by LDAP through Group Sync. You can allow access level overrides by calling this endpoint. @@ -298,8 +304,8 @@ POST /groups/:id/members/:user_id/override | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | | `user_id` | integer | yes | The user ID of the member | -```bash -curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/:id/members/:user_id/override +```shell +curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/groups/:id/members/:user_id/override" ``` Example response: @@ -320,7 +326,7 @@ Example response: ### Remove override for a member of a group -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4875) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4875) in GitLab 12.10. Sets the override flag to false and allows LDAP Group Sync to reset the access level to the LDAP-prescribed value. @@ -334,8 +340,8 @@ DELETE /groups/:id/members/:user_id/override | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user | | `user_id` | integer | yes | The user ID of the member | -```bash -curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/:id/members/:user_id/override +```shell +curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/groups/:id/members/:user_id/override" ``` Example response: @@ -369,10 +375,10 @@ DELETE /projects/:id/members/:user_id | `user_id` | integer | yes | The user ID of the member | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/members/:user_id -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/members/:user_id +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members/:user_id" +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/members/:user_id" ``` ## Give a group access to a project -Look at [share project with group](projects.md#share-project-with-group) +See [share project with group](projects.md#share-project-with-group) diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index c07e52451d2..746a79e1b8e 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -6,7 +6,7 @@ Configuration for approvals on all Merge Requests (MR) in the project. Must be a ### Get Configuration -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. You can request information about a project's approval configuration using the following endpoint: @@ -34,7 +34,7 @@ GET /projects/:id/approvals ### Change configuration -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. If you are allowed to, you can change approval configuration using the following endpoint: @@ -68,8 +68,8 @@ POST /projects/:id/approvals ### Get project-level rules -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. -> - `protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/issues/460) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> - `protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. You can request information about a project's approval rules using the following endpoint: @@ -168,7 +168,7 @@ GET /projects/:id/approval_rules ### Create project-level rule -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can create project approval rules using the following endpoint: @@ -270,7 +270,7 @@ POST /projects/:id/approval_rules ### Update project-level rule -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can update project approval rules using the following endpoint: @@ -375,7 +375,7 @@ PUT /projects/:id/approval_rules/:approval_rule_id ### Delete project-level rule -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can delete project approval rules using the following endpoint: @@ -393,7 +393,7 @@ DELETE /projects/:id/approval_rules/:approval_rule_id ### Change allowed approvers >**Note:** This API endpoint has been deprecated. Please use Approval Rule API instead. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. If you are allowed to, you can change approvers and approver groups using the following endpoint: @@ -505,7 +505,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/approvals ### Change approval configuration -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. If you are allowed to, you can change `approvals_required` using the following endpoint: @@ -542,7 +542,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/approvals ### Change allowed approvers for Merge Request >**Note:** This API endpoint has been deprecated. Please use Approval Rule API instead. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/183) in [GitLab Starter](https://about.gitlab.com/pricing/) 10.6. If you are allowed to, you can change approvers and approver groups using the following endpoint: @@ -613,7 +613,7 @@ PUT /projects/:id/merge_requests/:merge_request_iid/approvers ### Get the approval state of merge requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13712) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13712) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can request information about a merge request's approval state by using the following endpoint: @@ -685,7 +685,7 @@ This includes additional information about the users who have already approved ### Get merge request level rules -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13712) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13712) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can request information about a merge request's approval rules using the following endpoint: @@ -762,7 +762,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/approval_rules ### Create merge request level rule -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can create merge request approval rules using the following endpoint: @@ -846,7 +846,7 @@ will be used. ### Update merge request level rule -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can update merge request approval rules using the following endpoint: @@ -931,7 +931,7 @@ These are system generated rules. ### Delete merge request level rule -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.3. You can delete merge request approval rules using the following endpoint: diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 3834bb6fee3..41c0428485f 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -73,9 +73,8 @@ operation. If you are interested in the value of these fields from this endpoint, set the `with_merge_status_recheck` parameter to `true` in the query. NOTE: **Note:** -[Starting in GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/29984), -when `async_merge_request_check_mergeability` feature flag is enabled, the -mergeability (`merge_status`) of each merge request will be checked +[Starting in GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/29984), +the mergeability (`merge_status`) of each merge request will be checked asynchronously when a request is made to this endpoint. Poll this API endpoint to get updated status. This affects the `has_conflicts` property as it is dependent on the `merge_status`. It'll return `false` unless `merge_status` is @@ -554,9 +553,8 @@ Parameters: - `include_rebase_in_progress` (optional) - If `true` response includes whether a rebase operation is in progress NOTE: **Note:** -[Starting in GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/29984), -when `async_merge_request_check_mergeability` feature flag is enabled, the -mergeability (`merge_status`) of a merge request will be checked +[Starting in GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/29984), +the mergeability (`merge_status`) of a merge request will be checked asynchronously when a request is made to this endpoint. Poll this API endpoint to get updated status. This affects the `has_conflicts` property as it is dependent on the `merge_status`. It'll return `false` unless `merge_status` is @@ -1297,7 +1295,7 @@ DELETE /projects/:id/merge_requests/:merge_request_iid | `merge_request_iid` | integer | yes | The internal ID of the merge request | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/merge_requests/85 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/merge_requests/85" ``` ## Accept MR @@ -1655,7 +1653,7 @@ PUT /projects/:id/merge_requests/:merge_request_iid/rebase | `skip_ci` | boolean | no | Set to `true` to skip creating a CI pipeline | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/76/merge_requests/1/rebase +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/76/merge_requests/1/rebase" ``` This is an asynchronous request. The API will return a `202 Accepted` response @@ -1717,7 +1715,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/closes_issues | `merge_request_iid` | integer | yes | The internal ID of the merge request | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/76/merge_requests/1/closes_issues +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/76/merge_requests/1/closes_issues" ``` Example response when the GitLab issue tracker is used: @@ -1793,7 +1791,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/subscribe | `merge_request_iid` | integer | yes | The internal ID of the merge request | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/17/subscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/17/subscribe" ``` Example response: @@ -1944,7 +1942,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/unsubscribe | `merge_request_iid` | integer | yes | The internal ID of the merge request | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/17/unsubscribe +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/17/unsubscribe" ``` Example response: @@ -2095,7 +2093,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/todo | `merge_request_iid` | integer | yes | The internal ID of the merge request | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/27/todo +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/27/todo" ``` Example response: @@ -2212,7 +2210,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/versions | `merge_request_iid` | integer | yes | The internal ID of the merge request | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/merge_requests/1/versions +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/merge_requests/1/versions" ``` Example response: @@ -2254,7 +2252,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/versions/:version_id | `version_id` | integer | yes | The ID of the merge request diff version | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/merge_requests/1/versions/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/merge_requests/1/versions/1" ``` Example response: @@ -2322,7 +2320,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/time_estimate | `duration` | string | yes | The duration in human format. e.g: 3h30m | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/93/time_estimate?duration=3h30m +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/93/time_estimate?duration=3h30m" ``` Example response: @@ -2350,7 +2348,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/reset_time_estimate | `merge_request_iid` | integer | yes | The internal ID of a project's merge_request | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/93/reset_time_estimate +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/93/reset_time_estimate" ``` Example response: @@ -2379,7 +2377,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/add_spent_time | `duration` | string | yes | The duration in human format. e.g: 3h30m | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/93/add_spent_time?duration=1h +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/93/add_spent_time?duration=1h" ``` Example response: @@ -2407,7 +2405,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/reset_spent_time | `merge_request_iid` | integer | yes | The internal ID of a project's merge_request | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/93/reset_spent_time +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/93/reset_spent_time" ``` Example response: @@ -2433,7 +2431,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/time_stats | `merge_request_iid` | integer | yes | The internal ID of the merge request | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/93/time_stats +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/93/time_stats" ``` Example response: diff --git a/doc/api/merge_trains.md b/doc/api/merge_trains.md index d8446ed445f..3cfef3864ad 100644 --- a/doc/api/merge_trains.md +++ b/doc/api/merge_trains.md @@ -1,6 +1,12 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Merge Trains API **(PREMIUM)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36146) in GitLab 12.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36146) in GitLab 12.9. > - Using this API you can consume GitLab's [Merge Train](../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) entries. Every API call to merge trains must be authenticated with Developer or higher [permissions](../user/permissions.md). @@ -32,7 +38,7 @@ GET /projects/:id/merge_trains?scope=complete | `sort` | string | no | Return Merge Trains sorted in `asc` or `desc` order. Default is `desc`. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/merge_trains +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/merge_trains" ``` Example response: diff --git a/doc/api/metrics_dashboard_annotations.md b/doc/api/metrics_dashboard_annotations.md index 09187a096ef..05bf7156a7e 100644 --- a/doc/api/metrics_dashboard_annotations.md +++ b/doc/api/metrics_dashboard_annotations.md @@ -2,7 +2,7 @@ > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29089) in GitLab 12.10 behind a disabled feature flag. -Metrics dashboard annotations allow you to indicate events on your graphs at a single point in time or over a timespan. +Metrics dashboard annotations allow you to indicate events on your graphs at a single point in time or over a time span. ## Create a new annotation @@ -12,7 +12,7 @@ POST /clusters/:id/metrics_dashboard/annotations/ ``` NOTE: **Note:** -The value of `dashboard_path` will be treated as a CGI-escaped path, and automatically unescaped. +The value of `dashboard_path` will be treated as a CGI-escaped path, and automatically un-escaped. Parameters: @@ -24,7 +24,7 @@ Parameters: | `description` | string | yes | Description of the annotation. | ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/environments/1/metrics_dashboard/annotations \ +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/environments/1/metrics_dashboard/annotations" \ --data-urlencode "dashboard_path=.gitlab/dashboards/custom_metrics.yml" \ --data-urlencode "starting_at=2016-03-11T03:45:40Z" \ --data-urlencode "description=annotation description" diff --git a/doc/api/milestones.md b/doc/api/milestones.md index 5727a4b637f..b5702c7d6e0 100644 --- a/doc/api/milestones.md +++ b/doc/api/milestones.md @@ -1,5 +1,14 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Project milestones API +This page describes the project milestones API. +There's a separate [group milestones API](./group_milestones.md) page. + ## List project milestones Returns a list of project milestones. @@ -25,7 +34,7 @@ Parameters: | `search` | string | optional | Return only milestones with a title or description matching the provided string | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/milestones +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/milestones" ``` Example Response: @@ -135,7 +144,7 @@ Parameters: ## Promote project milestone to a group milestone -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53861) in GitLab 11.9 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53861) in GitLab 11.9 Only for users with Developer access to the group. @@ -150,7 +159,7 @@ Parameters: ## Get all burndown chart events for a single milestone **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4737) in GitLab 12.1 +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4737) in GitLab 12.1 Gets all burndown chart events for a single milestone. diff --git a/doc/api/namespaces.md b/doc/api/namespaces.md index 50b5f3f19cd..d1a2812bfb4 100644 --- a/doc/api/namespaces.md +++ b/doc/api/namespaces.md @@ -19,7 +19,7 @@ GET /namespaces Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/namespaces +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces" ``` Example response: @@ -85,7 +85,7 @@ GET /namespaces?search=foobar Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/namespaces?search=twitter +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces?search=twitter" ``` Example response: @@ -119,7 +119,7 @@ GET /namespaces/:id Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/namespaces/2 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces/2" ``` Example response: @@ -139,7 +139,7 @@ Example response: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/namespaces/group1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces/group1" ``` Example response: diff --git a/doc/api/notes.md b/doc/api/notes.md index 62e1fe44bae..74d941edec1 100644 --- a/doc/api/notes.md +++ b/doc/api/notes.md @@ -82,7 +82,7 @@ GET /projects/:id/issues/:issue_iid/notes?sort=asc&order_by=updated_at ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/notes +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/notes" ``` ### Get single issue note @@ -100,7 +100,7 @@ Parameters: - `note_id` (required) - The ID of an issue note ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/notes/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/notes/1" ``` ### Create new issue note @@ -119,7 +119,7 @@ Parameters: - `created_at` (optional) - Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires admin or project/group owner rights) ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/notes?body=note +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/notes?body=note" ``` ### Modify existing issue note @@ -138,7 +138,7 @@ Parameters: - `body` (required) - The content of a note. Limited to 1,000,000 characters. ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/notes?body=note +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/notes?body=note" ``` ### Delete an issue note @@ -158,7 +158,7 @@ Parameters: | `note_id` | integer | yes | The ID of a note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/notes/636 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/notes/636" ``` ## Snippets @@ -180,7 +180,7 @@ GET /projects/:id/snippets/:snippet_id/notes?sort=asc&order_by=updated_at | `order_by` | string | no | Return snippet notes ordered by `created_at` or `updated_at` fields. Default is `created_at` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/notes +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/notes" ``` ### Get single snippet note @@ -217,7 +217,7 @@ Parameters: ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/notes/11 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/notes/11" ``` ### Create new snippet note @@ -237,7 +237,7 @@ Parameters: - `created_at` (optional) - Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippet/11/notes?body=note +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippet/11/notes?body=note" ``` ### Modify existing snippet note @@ -256,7 +256,7 @@ Parameters: - `body` (required) - The content of a note. Limited to 1,000,000 characters. ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/11/notes?body=note +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/11/notes?body=note" ``` ### Delete a snippet note @@ -276,7 +276,7 @@ Parameters: | `note_id` | integer | yes | The ID of a note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippets/52/notes/1659 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippets/52/notes/1659" ``` ## Merge Requests @@ -298,7 +298,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/notes?sort=asc&order_by=upda | `order_by` | string | no | Return merge request notes ordered by `created_at` or `updated_at` fields. Default is `created_at` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes" ``` ### Get single merge request note @@ -340,7 +340,7 @@ Parameters: ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes/1" ``` ### Create new merge request note @@ -376,7 +376,7 @@ Parameters: - `body` (required) - The content of a note. Limited to 1,000,000 characters. ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes?body=note +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes?body=note" ``` ### Delete a merge request note @@ -396,7 +396,7 @@ Parameters: | `note_id` | integer | yes | The ID of a note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/7/notes/1602 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/7/notes/1602" ``` ## Epics **(ULTIMATE)** @@ -418,7 +418,7 @@ GET /groups/:id/epics/:epic_id/notes?sort=asc&order_by=updated_at | `order_by` | string | no | Return epic notes ordered by `created_at` or `updated_at` fields. Default is `created_at` | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/notes +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/notes" ``` ### Get single epic note @@ -458,7 +458,7 @@ Parameters: ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/11/notes/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/notes/1" ``` ### Create new epic note @@ -479,7 +479,7 @@ Parameters: | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippet/11/notes?body=note +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippet/11/notes?body=note" ``` ### Modify existing epic note @@ -500,7 +500,7 @@ Parameters: | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/snippet/11/notes?body=note +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/snippet/11/notes?body=note" ``` ### Delete an epic note @@ -520,5 +520,5 @@ Parameters: | `note_id` | integer | yes | The ID of a note | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/epics/52/notes/1659 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/52/notes/1659" ``` diff --git a/doc/api/notification_settings.md b/doc/api/notification_settings.md index 596365743fa..ffdf6d34832 100644 --- a/doc/api/notification_settings.md +++ b/doc/api/notification_settings.md @@ -43,7 +43,7 @@ GET /notification_settings ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/notification_settings +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/notification_settings" ``` Example response: @@ -64,7 +64,7 @@ PUT /notification_settings ``` ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/notification_settings?level=watch +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/notification_settings?level=watch" ``` | Attribute | Type | Required | Description | @@ -107,8 +107,8 @@ GET /projects/:id/notification_settings ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/notification_settings -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/8/notification_settings +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/notification_settings" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/8/notification_settings" ``` | Attribute | Type | Required | Description | @@ -133,8 +133,8 @@ PUT /projects/:id/notification_settings ``` ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/5/notification_settings?level=watch -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/8/notification_settings?level=custom&new_note=true +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/notification_settings?level=watch" +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/8/notification_settings?level=custom&new_note=true" ``` | Attribute | Type | Required | Description | diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index a146fdd0d0c..cc8b31ecf17 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -173,11 +173,14 @@ the following parameters: } ``` +Also you must use HTTP Basic authentication using the `client_id` and`client_secret` +values to authenticate the client that performs a request. + Example cURL request: ```shell echo 'grant_type=password&username=<your_username>&password=<your_password>' > auth.txt -curl --data "@auth.txt" --request POST https://gitlab.example.com/oauth/token +curl --data "@auth.txt" --user client_id:client_secret --request POST "https://gitlab.example.com/oauth/token" ``` Then, you'll receive the access token back in the response: @@ -190,6 +193,8 @@ Then, you'll receive the access token back in the response: } ``` +By default, the scope of the access token is `api`, which provides complete read/write access. + For testing, you can use the `oauth2` Ruby gem: ```ruby @@ -210,7 +215,7 @@ GET https://gitlab.example.com/api/v4/user?access_token=OAUTH-TOKEN or you can put the token to the Authorization header: ```shell -curl --header "Authorization: Bearer OAUTH-TOKEN" https://gitlab.example.com/api/v4/user +curl --header "Authorization: Bearer OAUTH-TOKEN" "https://gitlab.example.com/api/v4/user" ``` ## Retrieving the token information @@ -229,7 +234,7 @@ You must supply the access token, either: - In the Authorization header: ```shell - curl --header "Authorization: Bearer <OAUTH-TOKEN>" https://gitlab.example.com/oauth/token/info + curl --header "Authorization: Bearer <OAUTH-TOKEN>" "https://gitlab.example.com/oauth/token/info" ``` The following is an example response: diff --git a/doc/api/packages.md b/doc/api/packages.md index 784343d29fd..ca7113bc743 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Packages API **(PREMIUM)** This is the API docs of [GitLab Packages](../administration/packages/index.md). @@ -20,11 +26,11 @@ GET /projects/:id/packages | `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, or `type`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi` or `nuget`. (_Introduced in GitLab 12.9_) +| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, or `nuget`. (_Introduced in GitLab 12.9_) | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_Introduced in GitLab 12.9_) ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/packages +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/packages" ``` Example response: @@ -67,11 +73,11 @@ GET /groups/:id/packages | `exclude_subgroups` | boolean | false | If the parameter is included as true, packages from projects from subgroups are not listed. Default is `false`. | | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, `type`, or `project_path`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi` or `nuget`. (_Introduced in GitLab 12.9_) | +| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, or `nuget`. (_Introduced in GitLab 12.9_) | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30980) in GitLab 13.0_) ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/:id/packages?exclude_subgroups=true +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/packages?exclude_subgroups=true" ``` CAUTION: **Deprecation** @@ -156,7 +162,7 @@ GET /projects/:id/packages/:package_id | `package_id` | integer | yes | ID of a package. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/packages/:package_id +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/packages/:package_id" ``` CAUTION: **Deprecation** @@ -233,7 +239,7 @@ GET /projects/:id/packages/:package_id/package_files | `package_id` | integer | yes | ID of a package. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/packages/4/package_files +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/packages/4/package_files" ``` Example response: @@ -288,7 +294,7 @@ DELETE /projects/:id/packages/:package_id | `package_id` | integer | yes | ID of a package. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/packages/:package_id +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/packages/:package_id" ``` Can return the following status codes: diff --git a/doc/api/pages.md b/doc/api/pages.md index db39ab04d9d..fda4a70cbd9 100644 --- a/doc/api/pages.md +++ b/doc/api/pages.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Pages API Endpoints for managing [GitLab Pages](https://about.gitlab.com/stages-devops-lifecycle/pages/). @@ -17,5 +23,5 @@ DELETE /projects/:id/pages | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --request 'DELETE' --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/2/pages +curl --request 'DELETE' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/2/pages" ``` diff --git a/doc/api/pages_domains.md b/doc/api/pages_domains.md index 43bb5a9b774..1fddc79814f 100644 --- a/doc/api/pages_domains.md +++ b/doc/api/pages_domains.md @@ -1,3 +1,9 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Pages domains API Endpoints for connecting custom domain(s) and TLS certificates in [GitLab Pages](https://about.gitlab.com/stages-devops-lifecycle/pages/). @@ -13,7 +19,7 @@ GET /pages/domains ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/pages/domains +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/pages/domains" ``` ```json @@ -44,7 +50,7 @@ GET /projects/:id/pages/domains | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/pages/domains +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/pages/domains" ``` ```json @@ -81,7 +87,7 @@ GET /projects/:id/pages/domains/:domain | `domain` | string | yes | The custom domain indicated by the user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/pages/domains/www.domain.example +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/pages/domains/www.domain.example" ``` ```json @@ -92,7 +98,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```json @@ -126,15 +132,15 @@ POST /projects/:id/pages/domains | `key` | file/string | no | The certificate key in PEM format. | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" "https://gitlab.example.com/api/v4/projects/5/pages/domains" ``` ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" https://gitlab.example.com/api/v4/projects/5/pages/domains +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" "https://gitlab.example.com/api/v4/projects/5/pages/domains" ``` ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "auto_ssl_enabled=true" https://gitlab.example.com/api/v4/projects/5/pages/domains +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "auto_ssl_enabled=true" "https://gitlab.example.com/api/v4/projects/5/pages/domains" ``` ```json @@ -170,11 +176,11 @@ PUT /projects/:id/pages/domains/:domain ### Adding certificate ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```json @@ -194,7 +200,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certifi ### Enabling Let's Encrypt integration for Pages custom domains ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "auto_ssl_enabled=true" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "auto_ssl_enabled=true" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```json @@ -210,7 +216,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "auto_ss To remove the SSL certificate attached to the Pages domain, run: ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=" --form "key=" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=" --form "key=" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```json @@ -235,5 +241,5 @@ DELETE /projects/:id/pages/domains/:domain | `domain` | string | yes | The custom domain indicated by the user | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` diff --git a/doc/api/pipeline_schedules.md b/doc/api/pipeline_schedules.md index 36a18411ceb..dc16157ef4b 100644 --- a/doc/api/pipeline_schedules.md +++ b/doc/api/pipeline_schedules.md @@ -16,7 +16,7 @@ GET /projects/:id/pipeline_schedules | `scope` | string | no | The scope of pipeline schedules, one of: `active`, `inactive` | ```shell -curl --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules" ``` ```json @@ -57,7 +57,7 @@ GET /projects/:id/pipeline_schedules/:pipeline_schedule_id | `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | ```shell -curl --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" ``` ```json @@ -113,7 +113,7 @@ POST /projects/:id/pipeline_schedules | `active` | boolean | no | The activation of pipeline schedule. If false is set, the pipeline schedule will deactivated initially (default: `true`) | ```shell -curl --request POST --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form description="Build packages" --form ref="master" --form cron="0 1 * * 5" --form cron_timezone="UTC" --form active="true" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules" +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form description="Build packages" --form ref="master" --form cron="0 1 * * 5" --form cron_timezone="UTC" --form active="true" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules" ``` ```json @@ -158,7 +158,7 @@ PUT /projects/:id/pipeline_schedules/:pipeline_schedule_id | `active` | boolean | no | The activation of pipeline schedule. If false is set, the pipeline schedule will deactivated initially. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form cron="0 2 * * *" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form cron="0 2 * * *" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" ``` ```json @@ -203,7 +203,7 @@ POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/take_ownership | `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | ```shell -curl --request POST --header "PRIVATE-TOKEN: hf2CvZXB9w8Uc5pZKpSB" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/take_ownership" +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/take_ownership" ``` ```json @@ -248,7 +248,7 @@ DELETE /projects/:id/pipeline_schedules/:pipeline_schedule_id | `pipeline_schedule_id` | integer | yes | The pipeline schedule ID | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" ``` ```json @@ -281,7 +281,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gi ## Run a scheduled pipeline immediately -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201786) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201786) in GitLab 12.8. Trigger a new scheduled pipeline, which runs immediately. The next scheduled run of this pipeline is not affected. @@ -298,7 +298,7 @@ POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/play Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/pipeline_schedules/1/play +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/42/pipeline_schedules/1/play" ``` Example response: @@ -311,7 +311,7 @@ Example response: ## Pipeline schedule variables -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/34518) in GitLab 10.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34518) in GitLab 10.0. ## Create a new pipeline schedule variable @@ -330,7 +330,7 @@ POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file` | ```shell -curl --request POST --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form "key=NEW_VARIABLE" --form "value=new value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables" +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "key=NEW_VARIABLE" --form "value=new value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables" ``` ```json @@ -358,7 +358,7 @@ PUT /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables/:key | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file` | ```shell -curl --request PUT --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" --form "value=updated value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables/NEW_VARIABLE" +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "value=updated value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables/NEW_VARIABLE" ``` ```json @@ -384,7 +384,7 @@ DELETE /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables/:key | `key` | string | yes | The `key` of a variable | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: k5ESFgWY2Qf5xEvDcFxZ" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables/NEW_VARIABLE" +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables/NEW_VARIABLE" ``` ```json diff --git a/doc/api/project_aliases.md b/doc/api/project_aliases.md index 59c0ffee76d..d80decfe53c 100644 --- a/doc/api/project_aliases.md +++ b/doc/api/project_aliases.md @@ -1,6 +1,6 @@ # Project Aliases API **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3264) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3264) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. All methods require administrator authorization. diff --git a/doc/api/project_badges.md b/doc/api/project_badges.md index 4adeb50deca..86b1ba6ce19 100644 --- a/doc/api/project_badges.md +++ b/doc/api/project_badges.md @@ -25,7 +25,7 @@ GET /projects/:id/badges | `name` | string | no | Name of the badges to return (case-sensitive). | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/badges +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/badges" ``` Example response: @@ -67,7 +67,7 @@ GET /projects/:id/badges/:badge_id | `badge_id` | integer | yes | The badge ID | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/badges/:badge_id +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/badges/:badge_id" ``` Example response: @@ -98,7 +98,7 @@ POST /projects/:id/badges | `image_url` | string | yes | URL of the badge image | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&position=0" https://gitlab.example.com/api/v4/projects/:id/badges +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&position=0" "https://gitlab.example.com/api/v4/projects/:id/badges" ``` Example response: @@ -130,7 +130,7 @@ PUT /projects/:id/badges/:badge_id | `image_url` | string | no | URL of the badge image | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/badges/:badge_id +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/badges/:badge_id" ``` Example response: @@ -160,7 +160,7 @@ DELETE /projects/:id/badges/:badge_id | `badge_id` | integer | yes | The badge ID | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/badges/:badge_id +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/badges/:badge_id" ``` ## Preview a badge from a project @@ -178,7 +178,7 @@ GET /projects/:id/badges/render | `image_url` | string | yes | URL of the badge image | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/badges/render?link_url=http%3A%2F%2Fexample.com%2Fci_status.svg%3Fproject%3D%25%7Bproject_path%7D%26ref%3D%25%7Bdefault_branch%7D&image_url=https%3A%2F%2Fshields.io%2Fmy%2Fbadge +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/badges/render?link_url=http%3A%2F%2Fexample.com%2Fci_status.svg%3Fproject%3D%25%7Bproject_path%7D%26ref%3D%25%7Bdefault_branch%7D&image_url=https%3A%2F%2Fshields.io%2Fmy%2Fbadge" ``` Example response: diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index 79800af2f59..f7c8ffc8df6 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -22,7 +22,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/projects/26/clusters +curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters" ``` Example response: @@ -91,7 +91,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/projects/26/clusters/18 +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/26/clusters/18" ``` Example response: @@ -192,7 +192,7 @@ Parameters: Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/projects/26/clusters/user \ +curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters/user" \ -H "Accept: application/json" \ -H "Content-Type:application/json" \ -X POST --data '{"name":"cluster-5", "platform_kubernetes_attributes":{"api_url":"https://35.111.51.20","token":"12345","namespace":"cluster-5-namespace","ca_cert":"-----BEGIN CERTIFICATE-----\r\nhFiK1L61owwDQYJKoZIhvcNAQELBQAw\r\nLzEtMCsGA1UEAxMkZDA1YzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM4ZDBj\r\nMB4XDTE4MTIyNzIwMDM1MVoXDTIzMTIyNjIxMDM1MVowLzEtMCsGA1UEAxMkZDA1\r\nYzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM.......-----END CERTIFICATE-----"}}' @@ -289,7 +289,7 @@ through the ["Add existing cluster to project"](#add-existing-cluster-to-project Example request: ```shell -curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/projects/26/clusters/24 \ +curl --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters/24" \ -H "Content-Type:application/json" \ -X PUT --data '{"name":"new-cluster-name","domain":"new-domain.com","api_url":"https://new-api-url.com"}' ``` @@ -383,5 +383,5 @@ Parameters: Example request: ```shell -curl --request DELETE --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/projects/26/clusters/23 +curl --request DELETE --header 'Private-Token: <your_access_token>' "https://gitlab.example.com/api/v4/projects/26/clusters/23" ``` diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index ae2fbcec0ff..74902a22594 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -1,6 +1,6 @@ # Project import/export API -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41899) in GitLab 10.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41899) in GitLab 10.6. See also: @@ -31,7 +31,7 @@ POST /projects/:id/export | `upload[http_method]` | string | no | The HTTP method to upload the exported project. Only `PUT` and `POST` methods allowed. Default is `PUT` | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/export \ +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/export" \ --data "upload[http_method]=PUT" \ --data-urlencode "upload[url]=https://example-bucket.s3.eu-west-3.amazonaws.com/backup?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMBJHN2O62W8IELQ%2F20180312%2Feu-west-3%2Fs3%2Faws4_request&X-Amz-Date=20180312T110328Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=8413facb20ff33a49a147a0b4abcff4c8487cc33ee1f7e450c46e8f695569dbd" ``` @@ -58,7 +58,7 @@ GET /projects/:id/export | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/export +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/export" ``` Status can be one of: @@ -111,7 +111,7 @@ GET /projects/:id/export/download | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" --remote-header-name --remote-name https://gitlab.example.com/api/v4/projects/5/export/download +curl --header "PRIVATE-TOKEN: <your_access_token>" --remote-header-name --remote-name "https://gitlab.example.com/api/v4/projects/5/export/download" ``` ```shell @@ -142,7 +142,7 @@ The `file=` parameter must point to a file on your file system and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "path=api-project" --form "file=@/path/to/file" https://gitlab.example.com/api/v4/projects/import +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "path=api-project" --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/projects/import" ``` cURL doesn't support posting a file from a remote server. Importing a project from a remote server can be accomplished through something like the following: @@ -181,6 +181,10 @@ requests.post(url, headers=headers, data=data, files=files) } ``` +NOTE: **Note:** +The maximum import file size can be set by the Administrator, default is 50MB. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin UI](../user/admin_area/settings/account_and_limit_settings.md). + ## Import status Get the status of an import. @@ -194,7 +198,7 @@ GET /projects/:id/import | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/import +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/import" ``` Status can be one of: diff --git a/doc/api/project_repository_storage_moves.md b/doc/api/project_repository_storage_moves.md index 8df472f193f..c55d4a19feb 100644 --- a/doc/api/project_repository_storage_moves.md +++ b/doc/api/project_repository_storage_moves.md @@ -1,3 +1,10 @@ +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + # Project repository storage move API > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31285) in GitLab 13.0. @@ -16,7 +23,50 @@ are [paginated](README.md#pagination). Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://primary.example.com/api/v4/project_repository_storage_moves' +curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/project_repository_storage_moves' +``` + +Example response: + +```json +[ + { + "id": 1, + "created_at": "2020-05-07T04:27:17.234Z", + "state": "scheduled", + "source_storage_name": "default", + "destination_storage_name": "storage2", + "project": { + "id": 1, + "description": null, + "name": "project1", + "name_with_namespace": "John Doe2 / project1", + "path": "project1", + "path_with_namespace": "namespace1/project1", + "created_at": "2020-05-07T04:27:17.016Z" + } +] +``` + +## Retrieve all repository storage moves for a project + +```plaintext +GET /projects/:project_id/repository_storage_moves +``` + +By default, `GET` requests return 20 results at a time because the API results +are [paginated](README.md#pagination). + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `project_id` | integer | yes | ID of the project | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/1/repository_storage_moves' ``` Example response: @@ -44,19 +94,98 @@ Example response: ## Get a single project repository storage move ```plaintext -GET /project_repository_storage_moves/:id +GET /project_repository_storage_moves/:repository_storage_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `repository_storage_id` | integer | yes | ID of the project repository storage move | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/project_repository_storage_moves/1' +``` + +Example response: + +```json +{ + "id": 1, + "created_at": "2020-05-07T04:27:17.234Z", + "state": "scheduled", + "source_storage_name": "default", + "destination_storage_name": "storage2", + "project": { + "id": 1, + "description": null, + "name": "project1", + "name_with_namespace": "John Doe2 / project1", + "path": "project1", + "path_with_namespace": "namespace1/project1", + "created_at": "2020-05-07T04:27:17.016Z" +} +``` + +## Get a single repository storage move for a project + +```plaintext +GET /projects/:project_id/repository_storage_moves/:repository_storage_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `project_id` | integer | yes | ID of the project | +| `repository_storage_id` | integer | yes | ID of the project repository storage move | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/1/repository_storage_moves/1' +``` + +Example response: + +```json +{ + "id": 1, + "created_at": "2020-05-07T04:27:17.234Z", + "state": "scheduled", + "source_storage_name": "default", + "destination_storage_name": "storage2", + "project": { + "id": 1, + "description": null, + "name": "project1", + "name_with_namespace": "John Doe2 / project1", + "path": "project1", + "path_with_namespace": "namespace1/project1", + "created_at": "2020-05-07T04:27:17.016Z" +} +``` + +## Schedule a repository storage move for a project + +```plaintext +POST /projects/:project_id/repository_storage_moves ``` Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of the project repository storage move | +| `project_id` | integer | yes | ID of the project | +| `destination_storage_name` | string | yes | Name of the destination storage shard | Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://primary.example.com/api/v4/project_repository_storage_moves/1' +curl --request POST --header "PRIVATE_TOKEN: <your_access_token>" --header "Content-Type: application/json" \ +--data '{"destination_storage_name":"storage2"}' 'https://gitlab.example.com/api/v4/projects/1/repository_storage_moves' ``` Example response: diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index e435f87dcdb..e5dd85fb3bb 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -17,7 +17,7 @@ NOTE: **Note:** From July 2019, the `Internal` visibility setting is disabled for new projects, groups, and snippets on GitLab.com. Existing projects, groups, and snippets using the `Internal` visibility setting keep this setting. You can read more about the change in the -[relevant issue](https://gitlab.com/gitlab-org/gitlab/issues/12388). +[relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/12388). ## List snippets @@ -86,7 +86,7 @@ Parameters: Example request: ```shell -curl --request POST https://gitlab.com/api/v4/projects/:id/snippets \ +curl --request POST "https://gitlab.com/api/v4/projects/:id/snippets" \ --header "PRIVATE-TOKEN: <your access token>" \ --header "Content-Type: application/json" \ -d @snippet.json @@ -125,7 +125,7 @@ Parameters: Example request: ```shell -curl --request PUT https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id \ +curl --request PUT "https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id" \ --header "PRIVATE-TOKEN: <your_access_token>" \ --header "Content-Type: application/json" \ -d @snippet.json @@ -159,7 +159,7 @@ Parameters: Example request: ```shell -curl --request DELETE https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id \ +curl --request DELETE "https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id" \ --header "PRIVATE-TOKEN: <your_access_token>" ``` @@ -179,13 +179,13 @@ Parameters: Example request: ```shell -curl https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id/raw \ +curl "https://gitlab.com/api/v4/projects/:id/snippets/:snippet_id/raw" \ --header "PRIVATE-TOKEN: <your_access_token>" ``` ## Get user agent details -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/29508) in GitLab 9.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29508) in GitLab 9.4. Available only for admins. @@ -201,7 +201,7 @@ GET /projects/:id/snippets/:snippet_id/user_agent_detail Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/snippets/2/user_agent_detail +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/snippets/2/user_agent_detail" ``` Example response: diff --git a/doc/api/project_templates.md b/doc/api/project_templates.md index 0c1b483600f..e9658423b63 100644 --- a/doc/api/project_templates.md +++ b/doc/api/project_templates.md @@ -4,7 +4,7 @@ This API is a project-specific version of these endpoints: - [Dockerfile templates](templates/dockerfiles.md) - [Gitignore templates](templates/gitignores.md) -- [GitLab CI/CD Config templates](templates/gitlab_ci_ymls.md) +- [GitLab CI/CD Configuration templates](templates/gitlab_ci_ymls.md) - [Open source license templates](templates/licenses.md) It deprecates these endpoints, which will be removed for API version 5. @@ -16,7 +16,7 @@ Support will be added for [Issue and Merge Request templates](../user/project/de in a future release. Support for [Group-level file templates](../user/group/index.md#group-file-templates-premium) -**(PREMIUM)** was [added](https://gitlab.com/gitlab-org/gitlab/issues/5987) +**(PREMIUM)** was [added](https://gitlab.com/gitlab-org/gitlab/-/issues/5987) in GitLab 11.5 ## Get all templates of a particular type diff --git a/doc/api/project_vulnerabilities.md b/doc/api/project_vulnerabilities.md index 84bbc789b0c..c4f1adccd3c 100644 --- a/doc/api/project_vulnerabilities.md +++ b/doc/api/project_vulnerabilities.md @@ -1,14 +1,6 @@ # Project Vulnerabilities API **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10242) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. - -CAUTION: **Caution:** -This API is currently in development and is protected by a **disabled** -[feature flag](../development/feature_flags/index.md). -On a self-managed GitLab instance, an administrator can enable it by starting the Rails console -(`sudo gitlab-rails console`) and then running the following command: `Feature.enable(:first_class_vulnerabilities)`. -To test if the Vulnerabilities API was successfully enabled, run the following command: -`Feature.enabled?(:first_class_vulnerabilities)`. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10242) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. CAUTION: **Caution:** This API is in an alpha stage and considered unstable. @@ -43,8 +35,8 @@ GET /projects/:id/vulnerabilities | ------------- | -------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | -```bash -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/vulnerabilities +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/vulnerabilities" ``` Example response: @@ -133,7 +125,7 @@ its source Vulnerability Finding, or with these default values: | `confidence` | The `confidence` attribute of a Vulnerability Finding | ```shell -curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/vulnerabilities?finding_id=1 +curl --header POST "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/vulnerabilities?finding_id=1" ``` Example response: diff --git a/doc/api/projects.md b/doc/api/projects.md index a5bb3f9bebb..b9ba632cd9e 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Projects API ## Project visibility level @@ -152,6 +158,7 @@ When the user is authenticated and `simple` is not set this returns something li "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -242,6 +249,7 @@ When the user is authenticated and `simple` is not set this returns something li "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -309,7 +317,7 @@ GET /projects?custom_attributes[key]=value&custom_attributes[other_key]=other_va ### Pagination limits From GitLab 13.0, [offset-based pagination](README.md#offset-based-pagination) will be -[limited to 50,000 records](https://gitlab.com/gitlab-org/gitlab/issues/34565). +[limited to 50,000 records](https://gitlab.com/gitlab-org/gitlab/-/issues/34565). [Keyset pagination](README.md#keyset-based-pagination) will be required to retrieve projects beyond this limit. @@ -401,6 +409,7 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -491,6 +500,7 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -617,6 +627,7 @@ Example response: "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -702,6 +713,7 @@ Example response: "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -855,6 +867,7 @@ GET /projects/:id ], "repository_storage": "default", "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "printing_merge_requests_link_enabled": true, @@ -1038,6 +1051,7 @@ POST /projects | `import_url` | string | no | URL to import repository from | | `public_builds` | boolean | no | If `true`, jobs can be viewed by non-project-members | | `only_allow_merge_if_pipeline_succeeds` | boolean | no | Set whether merge requests can only be merged with successful jobs | +| `allow_merge_on_skipped_pipeline` | boolean | no | Set whether or not merge requests can be merged with skipped jobs | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | no | Set whether merge requests can only be merged when all the discussions are resolved | | `merge_method` | string | no | Set the [merge method](#project-merge-method) used | | `autoclose_referenced_issues` | boolean | no | Set whether auto-closing referenced issues on default branch | @@ -1107,6 +1121,7 @@ POST /projects/user/:user_id | `import_url` | string | no | URL to import repository from | | `public_builds` | boolean | no | If `true`, jobs can be viewed by non-project-members | | `only_allow_merge_if_pipeline_succeeds` | boolean | no | Set whether merge requests can only be merged with successful jobs | +| `allow_merge_on_skipped_pipeline` | boolean | no | Set whether or not merge requests can be merged with skipped jobs | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | no | Set whether merge requests can only be merged when all the discussions are resolved | | `merge_method` | string | no | Set the [merge method](#project-merge-method) used | | `autoclose_referenced_issues` | boolean | no | Set whether auto-closing referenced issues on default branch | @@ -1177,6 +1192,7 @@ PUT /projects/:id | `import_url` | string | no | URL to import repository from | | `public_builds` | boolean | no | If `true`, jobs can be viewed by non-project-members | | `only_allow_merge_if_pipeline_succeeds` | boolean | no | Set whether merge requests can only be merged with successful jobs | +| `allow_merge_on_skipped_pipeline` | boolean | no | Set whether or not merge requests can be merged with skipped jobs | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | no | Set whether merge requests can only be merged when all the discussions are resolved | | `merge_method` | string | no | Set the [merge method](#project-merge-method) used | | `autoclose_referenced_issues` | boolean | no | Set whether auto-closing referenced issues on default branch | @@ -1311,6 +1327,7 @@ Example responses: "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -1402,6 +1419,7 @@ Example response: "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -1492,6 +1510,7 @@ Example response: "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -1674,6 +1693,7 @@ Example response: "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -1783,6 +1803,7 @@ Example response: "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, + "allow_merge_on_skipped_pipeline": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": false, "request_access_enabled": false, @@ -1806,7 +1827,7 @@ Example response: This endpoint either: - Removes a project including all associated resources (issues, merge requests etc). -- From [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/32935) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a project for deletion. Actual +- From [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) on [Premium or Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a project for deletion. Actual deletion happens after number of days specified in [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). @@ -1820,7 +1841,7 @@ DELETE /projects/:id ## Restore project marked for deletion **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32935) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. Restores project marked for deletion. @@ -1851,7 +1872,7 @@ The `file=` parameter must point to a file on your filesystem and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" https://gitlab.example.com/api/v4/projects/5/uploads +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/5/uploads" ``` Returned object: @@ -1898,7 +1919,7 @@ DELETE /projects/:id/share/:group_id | `group_id` | integer | yes | The ID of the group | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/share/17 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/share/17" ``` ## Hooks @@ -1943,6 +1964,7 @@ GET /projects/:id/hooks/:hook_id "merge_requests_events": true, "tag_push_events": true, "note_events": true, + "confidential_note_events": true, "job_events": true, "pipeline_events": true, "wiki_page_events": true, @@ -1970,6 +1992,7 @@ POST /projects/:id/hooks | `merge_requests_events` | boolean | no | Trigger hook on merge requests events | | `tag_push_events` | boolean | no | Trigger hook on tag push events | | `note_events` | boolean | no | Trigger hook on note events | +| `confidential_note_events` | boolean | no | Trigger hook on confidential note events | | `job_events` | boolean | no | Trigger hook on job events | | `pipeline_events` | boolean | no | Trigger hook on pipeline events | | `wiki_page_events` | boolean | no | Trigger hook on wiki events | @@ -1996,6 +2019,7 @@ PUT /projects/:id/hooks/:hook_id | `merge_requests_events` | boolean | no | Trigger hook on merge requests events | | `tag_push_events` | boolean | no | Trigger hook on tag push events | | `note_events` | boolean | no | Trigger hook on note events | +| `confidential_note_events` | boolean | no | Trigger hook on confidential note events | | `job_events` | boolean | no | Trigger hook on job events | | `pipeline_events` | boolean | no | Trigger hook on pipeline events | | `wiki_events` | boolean | no | Trigger hook on wiki events | @@ -2061,7 +2085,7 @@ GET /projects | `sort` | string | no | Return requests sorted in `asc` or `desc` order | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects?search=test +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects?search=test" ``` ## Start the Housekeeping task for a Project @@ -2221,7 +2245,7 @@ POST /projects/:id/mirror/pull | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:id/mirror/pull +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/mirror/pull" ``` ## Project badges diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index de862109055..4206fe6a565 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -27,7 +27,7 @@ GET /projects/:id/protected_branches | `search` | string | no | Name or part of the name of protected branches to be searched for | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_branches' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_branches" ``` Example response: @@ -101,7 +101,7 @@ GET /projects/:id/protected_branches/:name | `name` | string | yes | The name of the branch or wildcard | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_branches/master' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_branches/master" ``` Example response: @@ -165,7 +165,7 @@ POST /projects/:id/protected_branches ``` ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_branches?name=*-stable&push_access_level=30&merge_access_level=30&unprotect_access_level=40' +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_branches?name=*-stable&push_access_level=30&merge_access_level=30&unprotect_access_level=40" ``` | Attribute | Type | Required | Description | @@ -251,7 +251,7 @@ Elements in the `allowed_to_push` / `allowed_to_merge` / `allowed_to_unprotect` form `{user_id: integer}`, `{group_id: integer}` or `{access_level: integer}`. Each user must have access to the project and each group must [have this project shared](../user/project/members/share_project_with_groups.md). These access levels allow [more granular control over protected branch access](../user/project/protected_branches.md#restricting-push-and-merge-access-to-certain-users-starter) and were [added to the API in](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3516) in GitLab 10.3 EE. ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_branches?name=*-stable&allowed_to_push%5B%5D%5Buser_id%5D=1' +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_branches?name=*-stable&allowed_to_push%5B%5D%5Buser_id%5D=1" ``` Example response: @@ -297,7 +297,7 @@ DELETE /projects/:id/protected_branches/:name ``` ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_branches/*-stable' +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_branches/*-stable" ``` | Attribute | Type | Required | Description | @@ -314,7 +314,7 @@ PATCH /projects/:id/protected_branches/:name ``` ```shell -curl --request PATCH --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_branches/feature-branch' +curl --request PATCH --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_branches/feature-branch" ``` | Attribute | Type | Required | Description | diff --git a/doc/api/protected_environments.md b/doc/api/protected_environments.md index dea1382af29..765b8d2364d 100644 --- a/doc/api/protected_environments.md +++ b/doc/api/protected_environments.md @@ -1,6 +1,6 @@ # Protected environments API **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30595) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30595) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. ## Valid access levels @@ -26,7 +26,7 @@ GET /projects/:id/protected_environments | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_environments/' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_environments/" ``` Example response: @@ -61,7 +61,7 @@ GET /projects/:id/protected_environments/:name | `name` | string | yes | The name of the protected environment | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_environments/production' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_environments/production" ``` Example response: @@ -89,7 +89,7 @@ POST /projects/:id/protected_environments ``` ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_environments?name=staging&deploy_access_levels%5B%5D%5Buser_id%5D=1' +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_environments?name=staging&deploy_access_levels%5B%5D%5Buser_id%5D=1" ``` | Attribute | Type | Required | Description | @@ -127,7 +127,7 @@ DELETE /projects/:id/protected_environments/:name ``` ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_environments/staging' +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_environments/staging" ``` | Attribute | Type | Required | Description | diff --git a/doc/api/protected_tags.md b/doc/api/protected_tags.md index 1d844a2c5c4..01de19f54ea 100644 --- a/doc/api/protected_tags.md +++ b/doc/api/protected_tags.md @@ -26,7 +26,7 @@ GET /projects/:id/protected_tags | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_tags' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_tags" ``` Example response: @@ -61,7 +61,7 @@ GET /projects/:id/protected_tags/:name | `name` | string | yes | The name of the tag or wildcard | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_tags/release-1-0' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_tags/release-1-0" ``` Example response: @@ -88,7 +88,7 @@ POST /projects/:id/protected_tags ``` ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_tags?name=*-stable&create_access_level=30' +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_tags?name=*-stable&create_access_level=30" ``` | Attribute | Type | Required | Description | @@ -120,7 +120,7 @@ DELETE /projects/:id/protected_tags/:name ``` ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/5/protected_tags/*-stable' +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/protected_tags/*-stable" ``` | Attribute | Type | Required | Description | diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 9c3cac3c64f..2c933061c37 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -1,9 +1,15 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Releases API -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41766) in GitLab 11.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41766) in GitLab 11.7. > - Using this API you can manipulate GitLab's [Release](../../user/project/releases/index.md) entries. > - For manipulating links as a release asset, see [Release Links API](links.md). -> - Release Evidences were [introduced](https://gitlab.com/gitlab-org/gitlab/issues/26019) in GitLab 12.5. +> - Release Evidences were [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.5. ## List Releases @@ -121,13 +127,15 @@ Example response: "id":2, "name":"awesome-v0.2.msi", "url":"http://192.168.10.15:3000/msi", - "external":true + "external":true, + "link_type":"other" }, { "id":1, "name":"awesome-v0.2.dmg", "url":"http://192.168.10.15:3000", - "external":true + "external":true, + "link_type":"other" } ], "evidence_file_path":"https://gitlab.example.com/root/awesome-app/-/releases/v0.2/evidence.json" @@ -323,7 +331,8 @@ Example response: "id":3, "name":"hoge", "url":"https://gitlab.example.com/root/awesome-app/-/tags/v0.11.1/binaries/linux-amd64", - "external":true + "external":true, + "link_type":"other" } ] }, @@ -357,13 +366,14 @@ POST /projects/:id/releases | `assets:links:name`| string | required by: `assets:links` | The name of the link. | | `assets:links:url` | string | required by: `assets:links` | The URL of the link. | | `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases.md). +| `assets:links:link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | `released_at` | datetime | no | The date when the release will be/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | Example request: ```shell curl --header 'Content-Type: application/json' --header "PRIVATE-TOKEN: gDybLx3yrUK_HLp3qPjS" \ - --data '{ "name": "New release", "tag_name": "v0.3", "description": "Super nice release", "milestones": ["v1.0", "v1.0-rc"], "assets": { "links": [{ "name": "hoge", "url": "https://google.com", "filepath": "/binaries/linux-amd64" }] } }' \ + --data '{ "name": "New release", "tag_name": "v0.3", "description": "Super nice release", "milestones": ["v1.0", "v1.0-rc"], "assets": { "links": [{ "name": "hoge", "url": "https://google.com", "filepath": "/binaries/linux-amd64", "link_type":"other" }] } }' \ --request POST https://gitlab.example.com/api/v4/projects/24/releases ``` @@ -465,7 +475,8 @@ Example response: "id":3, "name":"hoge", "url":"https://gitlab.example.com/root/awesome-app/-/tags/v0.11.1/binaries/linux-amd64", - "external":true + "external":true, + "link_type":"other" } ], "evidence_file_path":"https://gitlab.example.com/root/awesome-app/-/releases/v0.3/evidence.json" @@ -693,7 +704,7 @@ Example response: ## Upcoming Releases -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/38105) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/38105) in GitLab 12.1. A release with a `released_at` attribute set to a future date will be labeled an **Upcoming Release** in the UI: diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index f380ba5a1b2..35cb66e59a1 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -1,6 +1,12 @@ +--- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Release links API -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41766) in GitLab 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41766) in GitLab 11.7. Using this API you can manipulate GitLab's [Release](../../user/project/releases/index.md) links. For manipulating other Release assets, see [Release API](index.md). GitLab supports links to `http`, `https`, and `ftp` assets. @@ -32,13 +38,15 @@ Example response: "id":2, "name":"awesome-v0.2.msi", "url":"http://192.168.10.15:3000/msi", - "external":true + "external":true, + "link_type":"other" }, { "id":1, "name":"awesome-v0.2.dmg", "url":"http://192.168.10.15:3000", - "external":true + "external":true, + "link_type":"other" } ] ``` @@ -70,7 +78,8 @@ Example response: "id":1, "name":"awesome-v0.2.dmg", "url":"http://192.168.10.15:3000", - "external":true + "external":true, + "link_type":"other" } ``` @@ -87,7 +96,8 @@ POST /projects/:id/releases/:tag_name/assets/links | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../README.md#namespaced-path-encoding). | | `tag_name` | string | yes | The tag associated with the Release. | | `name` | string | yes | The name of the link. | -| `url` | string | yes | The URL of the link. | +| `url` | string | yes | The URL of the link. | +| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | Example request: @@ -106,7 +116,8 @@ Example response: "id":1, "name":"awesome-v0.2.dmg", "url":"http://192.168.10.15:3000", - "external":true + "external":true, + "link_type":"other" } ``` @@ -122,9 +133,10 @@ PUT /projects/:id/releases/:tag_name/assets/links/:link_id | ------------- | -------------- | -------- | --------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../README.md#namespaced-path-encoding). | | `tag_name` | string | yes | The tag associated with the Release. | -| `link_id` | integer | yes | The ID of the link. | +| `link_id` | integer | yes | The ID of the link. | | `name` | string | no | The name of the link. | -| `url` | string | no | The URL of the link. | +| `url` | string | no | The URL of the link. | +| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | NOTE: **NOTE** You have to specify at least one of `name` or `url` @@ -132,7 +144,7 @@ You have to specify at least one of `name` or `url` Example request: ```shell -curl --request PUT --data name="new name" --header "PRIVATE-TOKEN: n671WNGecHugsdEDPsyo" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" +curl --request PUT --data name="new name" --data link_type="runbook" --header "PRIVATE-TOKEN: n671WNGecHugsdEDPsyo" "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" ``` Example response: @@ -142,7 +154,8 @@ Example response: "id":1, "name":"new name", "url":"http://192.168.10.15:3000", - "external":true + "external":true, + "link_type":"runbook" } ``` @@ -173,6 +186,7 @@ Example response: "id":1, "name":"new name", "url":"http://192.168.10.15:3000", - "external":true + "external":true, + "link_type":"other" } ``` diff --git a/doc/api/remote_mirrors.md b/doc/api/remote_mirrors.md index e46a890cbd4..6495f6d8383 100644 --- a/doc/api/remote_mirrors.md +++ b/doc/api/remote_mirrors.md @@ -7,7 +7,7 @@ outlined below. ## List a project's remote mirrors -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/38121) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38121) in GitLab 12.9. Returns an Array of remote mirrors and their statuses: @@ -18,7 +18,7 @@ GET /projects/:id/remote_mirrors Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/42/remote_mirrors" ``` Example response: @@ -46,7 +46,7 @@ and password information. ## Create a remote mirror -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/24189) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24189) in GitLab 12.9. Create a remote mirror for a project. The mirror will be disabled by default. You can enable it by including the optional parameter `enabled` when creating it: @@ -64,7 +64,7 @@ POST /projects/:id/remote_mirrors Example request: ```shell -curl --request POST --data "url=https://username:token@example.com/gitlab/example.git" --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors' +curl --request POST --data "url=https://username:token@example.com/gitlab/example.git" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/42/remote_mirrors" ``` Example response: @@ -86,7 +86,7 @@ Example response: ## Update a remote mirror's attributes -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/38121) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38121) in GitLab 12.9. Toggle a remote mirror on or off, or change which types of branches are mirrored: @@ -105,7 +105,7 @@ PUT /projects/:id/remote_mirrors/:mirror_id Example request: ```shell -curl --request PUT --data "enabled=false" --header "PRIVATE-TOKEN: <your_access_token>" 'https://gitlab.example.com/api/v4/projects/42/remote_mirrors/101486' +curl --request PUT --data "enabled=false" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/42/remote_mirrors/101486" ``` Example response: diff --git a/doc/api/repositories.md b/doc/api/repositories.md index 440db06792c..7e601ec96ec 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -124,7 +124,7 @@ Parameters: - `sha` (optional) - The commit SHA to download. A tag, branch reference, or SHA can be used. This defaults to the tip of the default branch if not specified. For example: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.com/api/v4/projects/<project_id>/repository/archive?sha=<commit_sha> +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.com/api/v4/projects/<project_id>/repository/archive?sha=<commit_sha>" ``` ## Compare branches, tags or commits diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index a8b58d90c34..9d934a0f855 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -25,7 +25,7 @@ GET /projects/:id/repository/files/:file_path ``` ```shell -curl --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fmodels%2Fkey%2Erb?ref=master' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fmodels%2Fkey%2Erb?ref=master" ``` Example response: @@ -51,7 +51,7 @@ Parameters: - `ref` (required) - The name of branch, tag or commit NOTE: **Note:** -`blob_id` is the blob sha, see [repositories - Get a blob from repository](repositories.md#get-a-blob-from-repository) +`blob_id` is the blob SHA, see [repositories - Get a blob from repository](repositories.md#get-a-blob-from-repository) In addition to the `GET` method, you can also use `HEAD` to get just file metadata. @@ -60,7 +60,7 @@ HEAD /projects/:id/repository/files/:file_path ``` ```shell -curl --head --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fmodels%2Fkey%2Erb?ref=master' +curl --head --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fmodels%2Fkey%2Erb?ref=master" ``` Example response: @@ -89,7 +89,7 @@ GET /projects/:id/repository/files/:file_path/blame ``` ```shell -curl --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.example.com/api/v4/projects/13083/repository/files/path%2Fto%2Ffile.rb/blame?ref=master' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/path%2Fto%2Ffile.rb/blame?ref=master" ``` Example response: @@ -129,7 +129,7 @@ NOTE: **Note:** `HEAD` method return just file metadata as in [Get file from repository](repository_files.md#get-file-from-repository). ```shell -curl --head --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.example.com/api/v4/projects/13083/repository/files/path%2Fto%2Ffile.rb/blame?ref=master' +curl --head --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/path%2Fto%2Ffile.rb/blame?ref=master" ``` Example response: @@ -156,7 +156,7 @@ GET /projects/:id/repository/files/:file_path/raw ``` ```shell -curl --header 'PRIVATE-TOKEN: <your_access_token>' 'https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fmodels%2Fkey%2Erb/raw?ref=master' +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fmodels%2Fkey%2Erb/raw?ref=master" ``` Parameters: @@ -179,7 +179,7 @@ POST /projects/:id/repository/files/:file_path curl --request POST --header 'PRIVATE-TOKEN: <your_access_token>' --header "Content-Type: application/json" \ --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ "content": "some content", "commit_message": "create a new file"}' \ - 'https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb' + "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" ``` Example response: @@ -214,7 +214,7 @@ PUT /projects/:id/repository/files/:file_path curl --request PUT --header 'PRIVATE-TOKEN: <your_access_token>' --header "Content-Type: application/json" \ --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ "content": "some content", "commit_message": "update file"}' \ - 'https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb' + "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" ``` Example response: @@ -260,7 +260,7 @@ DELETE /projects/:id/repository/files/:file_path curl --request DELETE --header 'PRIVATE-TOKEN: <your_access_token>' --header "Content-Type: application/json" \ --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ "commit_message": "delete file"}' \ - 'https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb' + "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" ``` Parameters: diff --git a/doc/api/repository_submodules.md b/doc/api/repository_submodules.md index 2ae7afa35b2..40708f5bcb0 100644 --- a/doc/api/repository_submodules.md +++ b/doc/api/repository_submodules.md @@ -1,6 +1,6 @@ # Repository submodules API -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41213) in GitLab 11.5 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41213) in GitLab 11.5 ## Update existing submodule reference in repository diff --git a/doc/api/resource_label_events.md b/doc/api/resource_label_events.md index 20f48674932..275614a1449 100644 --- a/doc/api/resource_label_events.md +++ b/doc/api/resource_label_events.md @@ -65,7 +65,7 @@ GET /projects/:id/issues/:issue_iid/resource_label_events ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/resource_label_events +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_label_events" ``` ### Get single issue label event @@ -85,7 +85,7 @@ Parameters: | `resource_label_event_id` | integer | yes | The ID of a label event | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/issues/11/resource_label_events/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_label_events/1" ``` ## Epics **(ULTIMATE)** @@ -151,7 +151,7 @@ GET /groups/:id/epics/:epic_id/resource_label_events ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/11/resource_label_events +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/11/resource_label_events" ``` ### Get single epic label event @@ -171,7 +171,7 @@ Parameters: | `resource_label_event_id` | integer | yes | The ID of a label event | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/1/epics/11/resource_label_events/107 +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/epics/11/resource_label_events/107" ``` ## Merge requests @@ -237,7 +237,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/resource_label_events ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_label_events +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_label_events" ``` ### Get single merge request label event @@ -257,5 +257,5 @@ Parameters: | `resource_label_event_id` | integer | yes | The ID of a label event | ```shell -curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_label_events/120 +curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_label_events/120" ``` diff --git a/doc/api/resource_milestone_events.md b/doc/api/resource_milestone_events.md new file mode 100644 index 00000000000..695687ada6d --- /dev/null +++ b/doc/api/resource_milestone_events.md @@ -0,0 +1,224 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Resource milestone events API + +Resource milestone events keep track of what happens to GitLab [issues](../user/project/issues/), +[merge requests](../user/project/merge_requests/), and [epics](../user/group/epics/). + +Use them to track which milestone was added or removed, who did it, and when it happened. + +## Issues + +### List project issue milestone events + +Gets a list of all milestone events for a single issue. + +```plaintext +GET /projects/:id/issues/:issue_iid/resource_milestone_events +``` + +| Attribute | Type | Required | Description | +| ----------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `issue_iid` | integer | yes | The IID of an issue | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_milestone_events" +``` + +Example response: + +```json +[ + { + "id": 142, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-20T13:38:20.077Z", + "resource_type": "Issue", + "resource_id": 253, + "milestone": { + "id": 61, + "iid": 9, + "project_id": 7, + "title": "v1.2", + "description": "Ipsum Lorem", + "state": "active", + "created_at": "2020-01-27T05:07:12.573Z", + "updated_at": "2020-01-27T05:07:12.573Z", + "due_date": null, + "start_date": null, + "web_url": "http://gitlab.example.com:3000/group/project/-/milestones/9" + }, + "action": "add" + }, + { + "id": 143, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-21T14:38:20.077Z", + "resource_type": "Issue", + "resource_id": 253, + "milestone": { + "id": 61, + "iid": 9, + "project_id": 7, + "title": "v1.2", + "description": "Ipsum Lorem", + "state": "active", + "created_at": "2020-01-27T05:07:12.573Z", + "updated_at": "2020-01-27T05:07:12.573Z", + "due_date": null, + "start_date": null, + "web_url": "http://gitlab.example.com:3000/group/project/-/milestones/9" + }, + "action": "remove" + } +] +``` + +### Get single issue milestone event + +Returns a single milestone event for a specific project issue + +```plaintext +GET /projects/:id/issues/:issue_iid/resource_milestone_events/:resource_milestone_event_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| ----------------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the project | +| `issue_iid` | integer | yes | The IID of an issue | +| `resource_milestone_event_id` | integer | yes | The ID of a milestone event | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/issues/11/resource_milestone_events/1" +``` + +## Merge requests + +### List project merge request milestone events + +Gets a list of all milestone events for a single merge request. + +```plaintext +GET /projects/:id/merge_requests/:merge_request_iid/resource_milestone_events +``` + +| Attribute | Type | Required | Description | +| ------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the project | +| `merge_request_iid` | integer | yes | The IID of a merge request | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_milestone_events" +``` + +Example response: + +```json +[ + { + "id": 142, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-20T13:38:20.077Z", + "resource_type": "MergeRequest", + "resource_id": 142, + "milestone": { + "id": 61, + "iid": 9, + "project_id": 7, + "title": "v1.2", + "description": "Ipsum Lorem", + "state": "active", + "created_at": "2020-01-27T05:07:12.573Z", + "updated_at": "2020-01-27T05:07:12.573Z", + "due_date": null, + "start_date": null, + "web_url": "http://gitlab.example.com:3000/group/project/-/milestones/9" + }, + "action": "add" + }, + { + "id": 143, + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://gitlab.example.com/root" + }, + "created_at": "2018-08-21T14:38:20.077Z", + "resource_type": "MergeRequest", + "resource_id": 142, + "milestone": { + "id": 61, + "iid": 9, + "project_id": 7, + "title": "v1.2", + "description": "Ipsum Lorem", + "state": "active", + "created_at": "2020-01-27T05:07:12.573Z", + "updated_at": "2020-01-27T05:07:12.573Z", + "due_date": null, + "start_date": null, + "web_url": "http://gitlab.example.com:3000/group/project/-/milestones/9" + }, + "action": "remove" + } +] +``` + +### Get single merge request milestone event + +Returns a single milestone event for a specific project merge request + +```plaintext +GET /projects/:id/merge_requests/:merge_request_iid/resource_milestone_events/:resource_milestone_event_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| ----------------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of a merge request | +| `resource_milestone_event_id` | integer | yes | The ID of a milestone event | + +Example request: + +```shell +curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/resource_milestone_events/120" +``` diff --git a/doc/api/runners.md b/doc/api/runners.md index 5db1f116f6c..4cda4b723f5 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -1,3 +1,9 @@ +--- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Runners API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2640) in GitLab 8.5 diff --git a/doc/api/scim.md b/doc/api/scim.md index 4300c9efa3d..7c8da37a949 100644 --- a/doc/api/scim.md +++ b/doc/api/scim.md @@ -86,7 +86,7 @@ Parameters: Example request: ```shell -curl 'https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl "https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Example response: @@ -130,7 +130,7 @@ Parameters: Example request: ```shell -curl --verbose --request POST 'https://example.gitlab.com/api/scim/v2/groups/test_group/Users' --data '{"externalId":"test_uid","active":null,"userName":"username","emails":[{"primary":true,"type":"work","value":"name@example.com"}],"name":{"formatted":"Test User","familyName":"User","givenName":"Test"},"schemas":["urn:ietf:params:scim:schemas:core:2.0:User"],"meta":{"resourceType":"User"}}' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl --verbose --request POST "https://example.gitlab.com/api/scim/v2/groups/test_group/Users" --data '{"externalId":"test_uid","active":null,"userName":"username","emails":[{"primary":true,"type":"work","value":"name@example.com"}],"name":{"formatted":"Test User","familyName":"User","givenName":"Test"},"schemas":["urn:ietf:params:scim:schemas:core:2.0:User"],"meta":{"resourceType":"User"}}' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Example response: @@ -184,7 +184,7 @@ Parameters: Example request: ```shell -curl --verbose --request PATCH 'https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2' --data '{ "Operations": [{"op":"Add","path":"name.formatted","value":"New Name"}] }' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl --verbose --request PATCH "https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --data '{ "Operations": [{"op":"Add","path":"name.formatted","value":"New Name"}] }' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Returns an empty response with a `204` status code if successful. @@ -207,7 +207,7 @@ Parameters: Example request: ```shell -curl --verbose --request DELETE 'https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2' --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" +curl --verbose --request DELETE "https://example.gitlab.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer <your_scim_token>" --header "Content-Type: application/scim+json" ``` Returns an empty response with a `204` status code if successful. diff --git a/doc/api/search.md b/doc/api/search.md index 7940a2fa4e3..9c4eef7dfe8 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -1,6 +1,6 @@ # Search API -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41763) in GitLab 10.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41763) in GitLab 10.5. Every API call to search must be authenticated. @@ -26,7 +26,7 @@ The response depends on the requested scope. ### Scope: projects ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=projects&search=flight +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=projects&search=flight" ``` Example response: @@ -57,7 +57,7 @@ Example response: ### Scope: issues ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=issues&search=file +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=issues&search=file" ``` Example response: @@ -122,7 +122,7 @@ Example response: ### Scope: merge_requests ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=merge_requests&search=file +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=merge_requests&search=file" ``` Example response: @@ -200,7 +200,7 @@ Example response: ### Scope: milestones ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=milestones&search=release +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=milestones&search=release" ``` Example response: @@ -225,7 +225,7 @@ Example response: ### Scope: snippet_titles ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=snippet_titles&search=sample +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=snippet_titles&search=sample" ``` Example response: @@ -258,7 +258,7 @@ Example response: This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=wiki_blobs&search=bye +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=wiki_blobs&search=bye" ``` Example response: @@ -279,14 +279,14 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: commits **(STARTER)** This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=commits&search=bye +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=commits&search=bye" ``` Example response: @@ -329,7 +329,7 @@ to use a filter simply include it in your query like so: `a query filename:some_ You may use wildcards (`*`) to use glob matching. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=blobs&search=installation +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=blobs&search=installation" ``` Example response: @@ -350,12 +350,12 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: users ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/search?scope=users&search=doe +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=users&search=doe" ``` Example response: @@ -398,7 +398,7 @@ The response depends on the requested scope. ### Scope: projects ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/3/search?scope=projects&search=flight +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=projects&search=flight" ``` Example response: @@ -429,7 +429,7 @@ Example response: ### Scope: issues ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/3/search?scope=issues&search=file +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=issues&search=file" ``` Example response: @@ -494,7 +494,7 @@ Example response: ### Scope: merge_requests ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/3/search?scope=merge_requests&search=file +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=merge_requests&search=file" ``` Example response: @@ -572,7 +572,7 @@ Example response: ### Scope: milestones ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/3/search?scope=milestones&search=release +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=milestones&search=release" ``` Example response: @@ -599,7 +599,7 @@ Example response: This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/6/search?scope=wiki_blobs&search=bye +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=wiki_blobs&search=bye" ``` Example response: @@ -620,14 +620,14 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: commits **(STARTER)** This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/6/search?scope=commits&search=bye +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=commits&search=bye" ``` Example response: @@ -670,7 +670,7 @@ to use a filter simply include it in your query like so: `a query filename:some_ You may use wildcards (`*`) to use glob matching. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/6/search?scope=blobs&search=installation +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=blobs&search=installation" ``` Example response: @@ -691,12 +691,12 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: users ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/3/search?scope=users&search=doe +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=users&search=doe" ``` Example response: @@ -738,7 +738,7 @@ The response depends on the requested scope. ### Scope: issues ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/12/search?scope=issues&search=file +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/12/search?scope=issues&search=file" ``` Example response: @@ -803,7 +803,7 @@ Example response: ### Scope: merge_requests ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/6/search?scope=merge_requests&search=file +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=merge_requests&search=file" ``` Example response: @@ -881,7 +881,7 @@ Example response: ### Scope: milestones ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/12/search?scope=milestones&search=release +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/12/search?scope=milestones&search=release" ``` Example response: @@ -906,7 +906,7 @@ Example response: ### Scope: notes ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" ``` Example response: @@ -955,7 +955,7 @@ results: times in the content. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/6/search?scope=wiki_blobs&search=bye +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=wiki_blobs&search=bye" ``` Example response: @@ -976,12 +976,12 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: commits ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/6/search?scope=commits&search=bye +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=commits&search=bye" ``` Example response: @@ -1028,7 +1028,7 @@ Blobs searches are performed on both filenames and contents. Search results: times in the content. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/6/search?scope=blobs&search=installation&ref=feature +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=blobs&search=installation&ref=feature" ``` Example response: @@ -1049,12 +1049,12 @@ Example response: ] ``` -**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521). +**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). ### Scope: users ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/6/search?scope=users&search=doe +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=users&search=doe" ``` Example response: diff --git a/doc/api/services.md b/doc/api/services.md index d435dffa651..02048a27c1b 100644 --- a/doc/api/services.md +++ b/doc/api/services.md @@ -714,7 +714,7 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `recipients` | string | true | Recipients/channels separated by whitespaces | -| `default_irc_uri` | string | false | irc://irc.network.net:6697/ | +| `default_irc_uri` | string | false | `irc://irc.network.net:6697/` | | `server_host` | string | false | localhost | | `server_port` | integer | false | 6659 | | `colorize_messages` | boolean | false | Colorize messages | @@ -1007,6 +1007,8 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `api_url` | string | true | Prometheus API Base URL. For example, `http://prometheus.example.com/`. | +| `google_iap_audience_client_id` | string | false | Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com) | +| `google_iap_service_account_json` | string | false | credentials.json file for your service account, like { "type": "service_account", "project_id": ... } | ### Delete Prometheus service @@ -1277,7 +1279,7 @@ A continuous integration and build server Set JetBrains TeamCity CI service for a project. -> The build configuration in Teamcity must use the build format number %build.vcs.number% you will also want to configure monitoring of all branches so merge requests build, that setting is in the vsc root advanced settings. +> The build configuration in TeamCity must use the build format number `%build.vcs.number%` you will also want to configure monitoring of all branches so merge requests build, that setting is in the VSC root advanced settings. ```plaintext PUT /projects/:id/services/teamcity diff --git a/doc/api/settings.md b/doc/api/settings.md index f63d126742a..78d992cff58 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -15,7 +15,7 @@ GET /application/settings ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/settings +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/settings" ``` Example response: @@ -30,6 +30,7 @@ Example response: "password_authentication_enabled_for_web" : true, "after_sign_out_path" : null, "max_attachment_size" : 10, + "max_import_size": 50, "user_oauth_applications" : true, "updated_at" : "2016-01-04T15:44:55.176Z", "session_expire_delay" : 10080, @@ -99,7 +100,7 @@ PUT /application/settings ``` ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/settings?signup_enabled=false&default_project_visibility=internal +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/settings?signup_enabled=false&default_project_visibility=internal" ``` Example response: @@ -118,6 +119,7 @@ Example response: "default_branch_protection": 2, "restricted_visibility_levels": [], "max_attachment_size": 10, + "max_import_size": 50, "session_expire_delay": 10080, "default_ci_config_path" : null, "default_project_visibility": "internal", @@ -280,6 +282,7 @@ are listed in the descriptions of the relevant settings. | `local_markdown_version` | integer | no | Increase this value when any cached Markdown should be invalidated. | | `max_artifacts_size` | integer | no | Maximum artifacts size in MB | | `max_attachment_size` | integer | no | Limit attachment size in MB | +| `max_import_size` | integer | no | Maximum import size in MB. 0 for unlimited. Default = 50 | | `max_pages_size` | integer | no | Maximum size of pages repositories in MB | | `max_personal_access_token_lifetime` | integer | no | **(ULTIMATE ONLY)** Maximum allowable lifetime for personal access tokens in days | | `metrics_method_call_threshold` | integer | no | A method call is only tracked when it takes longer than the given amount of milliseconds. | @@ -335,6 +338,8 @@ are listed in the descriptions of the relevant settings. | `sourcegraph_enabled` | boolean | no | Enables Sourcegraph integration. Default is `false`. **If enabled, requires** `sourcegraph_url`. | | `sourcegraph_url` | string | required by: `sourcegraph_enabled` | The Sourcegraph instance URL for integration. | | `sourcegraph_public_only` | boolean | no | Blocks Sourcegraph from being loaded on private and internal projects. Default is `true`. | +| `spam_check_endpoint_enabled` | boolean | no | Enables Spam Check via external API endpoint. Default is `false`. | +| `spam_check_endpoint_url` | string | no | URL of the external Spam Check service endpoint. | | `terminal_max_session_time` | integer | no | Maximum time for web terminal websocket connection (in seconds). Set to `0` for unlimited time. | | `terms` | text | required by: `enforce_terms` | (**Required by:** `enforce_terms`) Markdown content for the ToS. | | `throttle_authenticated_api_enabled` | boolean | no | (**If enabled, requires:** `throttle_authenticated_api_period_in_seconds` and `throttle_authenticated_api_requests_per_period`) Enable authenticated API request rate limit. Helps reduce request volume (for example, from crawlers or abusive bots). | diff --git a/doc/api/sidekiq_metrics.md b/doc/api/sidekiq_metrics.md index 5350feff4e3..8523ac88e00 100644 --- a/doc/api/sidekiq_metrics.md +++ b/doc/api/sidekiq_metrics.md @@ -15,7 +15,7 @@ GET /sidekiq/queue_metrics ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/sidekiq/queue_metrics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/sidekiq/queue_metrics" ``` Example response: @@ -40,7 +40,7 @@ GET /sidekiq/process_metrics ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/sidekiq/process_metrics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/sidekiq/process_metrics" ``` Example response: @@ -82,7 +82,7 @@ GET /sidekiq/job_stats ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/sidekiq/job_stats +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/sidekiq/job_stats" ``` Example response: @@ -107,7 +107,7 @@ GET /sidekiq/compound_metrics ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/sidekiq/compound_metrics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/sidekiq/compound_metrics" ``` Example response: diff --git a/doc/api/snippets.md b/doc/api/snippets.md index e2e39de412b..1aa3eecfd29 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -28,7 +28,7 @@ GET /snippets Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/snippets +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/snippets" ``` Example response: @@ -95,7 +95,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/snippets/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/snippets/1" ``` Example response: @@ -141,7 +141,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/snippets/1/raw +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/snippets/1/raw" ``` Example response: @@ -177,8 +177,8 @@ Example request: curl --request POST \ --data '{"title": "This is a snippet", "content": "Hello world", "description": "Hello World snippet", "file_name": "test.txt", "visibility": "internal" }' \ --header 'Content-Type: application/json' \ - --header "PRIVATE-TOKEN: valid_api_token" \ - https://gitlab.example.com/api/v4/snippets + --header "PRIVATE-TOKEN: <your_access_token>" \ + "https://gitlab.example.com/api/v4/snippets" ``` Example response: @@ -235,8 +235,8 @@ Example request: curl --request PUT \ --data '{"title": "foo", "content": "bar"}' \ --header 'Content-Type: application/json' \ - --header "PRIVATE-TOKEN: valid_api_token" \ - https://gitlab.example.com/api/v4/snippets/1 + --header "PRIVATE-TOKEN: <your_access_token>" \ + "https://gitlab.example.com/api/v4/snippets/1" ``` Example response: @@ -310,7 +310,7 @@ Parameters: Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/snippets/public?per_page=2&page=1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/snippets/public?per_page=2&page=1" ``` Example response: @@ -375,7 +375,7 @@ GET /snippets/:id/user_agent_detail Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/snippets/1/user_agent_detail +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/snippets/1/user_agent_detail" ``` Example response: diff --git a/doc/api/statistics.md b/doc/api/statistics.md index 883a7640cf8..890c6f68898 100644 --- a/doc/api/statistics.md +++ b/doc/api/statistics.md @@ -13,7 +13,7 @@ GET /application/statistics ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/statistics +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/statistics" ``` Example response: diff --git a/doc/api/suggestions.md b/doc/api/suggestions.md index 84bafd3c1ea..e3bbcaa51c3 100644 --- a/doc/api/suggestions.md +++ b/doc/api/suggestions.md @@ -16,7 +16,7 @@ PUT /suggestions/:id/apply | `id` | integer/string | yes | The ID of a suggestion | ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/suggestions/5/apply +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/suggestions/5/apply" ``` Example response: diff --git a/doc/api/system_hooks.md b/doc/api/system_hooks.md index cd69a6a6b34..3e0d2151428 100644 --- a/doc/api/system_hooks.md +++ b/doc/api/system_hooks.md @@ -18,7 +18,7 @@ GET /hooks Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/hooks +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/hooks" ``` Example response: @@ -92,7 +92,7 @@ GET /hooks/:id Example request: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/hooks/2 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/hooks/2" ``` Example response: @@ -123,5 +123,5 @@ DELETE /hooks/:id Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/hooks/2 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/hooks/2" ``` diff --git a/doc/api/tags.md b/doc/api/tags.md index 0a0490e072e..569271d6206 100644 --- a/doc/api/tags.md +++ b/doc/api/tags.md @@ -19,7 +19,7 @@ Parameters: | `sort` | string | no | Return tags sorted in `asc` or `desc` order. Default is `desc` | | `search` | string | no | Return list of tags matching the search criteria. You can use `^term` and `term$` to find tags that begin and end with `term` respectively. | -> Support for `search` was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/54401) in GitLab 11.8. +> Support for `search` was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54401) in GitLab 11.8. ```json [ @@ -69,7 +69,7 @@ Parameters: | `tag_name` | string | yes | The name of the tag | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/repository/tags/v1.0.0 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/repository/tags/v1.0.0" ``` Example Response: diff --git a/doc/api/todos.md b/doc/api/todos.md index cd6f2468cc6..9d56522c5b8 100644 --- a/doc/api/todos.md +++ b/doc/api/todos.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Todos API > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3188) in GitLab 8.10. @@ -23,7 +29,7 @@ Parameters: | `type` | string | no | The type of a todo. Can be either `Issue` or `MergeRequest` | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/todos +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/todos" ``` Example Response: @@ -197,7 +203,7 @@ Parameters: | `id` | integer | yes | The ID of a todo | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/todos/130/mark_as_done +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/todos/130/mark_as_done" ``` Example Response: @@ -288,5 +294,5 @@ POST /todos/mark_as_done ``` ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/todos/mark_as_done +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/todos/mark_as_done" ``` diff --git a/doc/api/users.md b/doc/api/users.md index 28d233ce6b3..6ac1cd089e7 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -70,12 +70,12 @@ Username search is case insensitive. GET /users ``` -| Attribute | Type | Required | Description | -| ------------ | ------ | -------- | ----------- | -| `order_by` | string | no | Return users ordered by `id`, `name`, `username`, `created_at`, or `updated_at` fields. Default is `id` | -| `sort` | string | no | Return users sorted in `asc` or `desc` order. Default is `desc` | -| `two_factor` | string | no | Filter users by Two-factor authentication. Filter values are `enabled` or `disabled`. By default it returns all users | -| `without_projects` | boolean | no | Filter users without projects. Default is `false` | +| Attribute | Type | Required | Description | +| ------------------ | ------- | -------- | --------------------------------------------------------------------------------------------------------------------- | +| `order_by` | string | no | Return users ordered by `id`, `name`, `username`, `created_at`, or `updated_at` fields. Default is `id` | +| `sort` | string | no | Return users sorted in `asc` or `desc` order. Default is `desc` | +| `two_factor` | string | no | Filter users by Two-factor authentication. Filter values are `enabled` or `disabled`. By default it returns all users | +| `without_projects` | boolean | no | Filter users without projects. Default is `false` | ```json [ @@ -104,6 +104,7 @@ GET /users "color_scheme_id": 2, "projects_limit": 100, "current_sign_in_at": "2012-06-02T06:36:55Z", + "note": "DMCA Request: 2018-11-05 | DMCA Violation | Abuse | https://gitlab.zendesk.com/agent/tickets/123", "identities": [ {"provider": "github", "extern_uid": "2435223452345"}, {"provider": "bitbucket", "extern_uid": "john.smith"}, @@ -154,7 +155,7 @@ GET /users ] ``` -Users on GitLab [Starter, Bronze, or higher](https://about.gitlab.com/pricing/) will also see the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit`, and `note` parameters. +Users on GitLab [Starter, Bronze, or higher](https://about.gitlab.com/pricing/) will also see the `shared_runners_minutes_limit`, and `extra_shared_runners_minutes_limit` parameters. ```json [ @@ -163,7 +164,6 @@ Users on GitLab [Starter, Bronze, or higher](https://about.gitlab.com/pricing/) ... "shared_runners_minutes_limit": 133, "extra_shared_runners_minutes_limit": 133, - "note": "DMCA Request: 2018-11-05 | DMCA Violation | Abuse | https://gitlab.zendesk.com/agent/tickets/123", ... } ] @@ -296,6 +296,7 @@ Example Responses: "color_scheme_id": 2, "projects_limit": 100, "current_sign_in_at": "2012-06-02T06:36:55Z", + "note": "DMCA Request: 2018-11-05 | DMCA Violation | Abuse | https://gitlab.zendesk.com/agent/tickets/123", "identities": [ {"provider": "github", "extern_uid": "2435223452345"}, {"provider": "bitbucket", "extern_uid": "john.smith"}, @@ -316,7 +317,7 @@ Example Responses: NOTE: **Note:** The `plan` and `trial` parameters are only available on GitLab Enterprise Edition. Users on GitLab [Starter, Bronze, or higher](https://about.gitlab.com/pricing/) will also see -the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit`, and `note` parameters. +the `shared_runners_minutes_limit`, and `extra_shared_runners_minutes_limit` parameters. ```json { @@ -324,7 +325,6 @@ the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit`, and `n "username": "john_smith", "shared_runners_minutes_limit": 133, "extra_shared_runners_minutes_limit": 133, - "note": "DMCA Request: 2018-11-05 | DMCA Violation | Abuse | https://gitlab.zendesk.com/agent/tickets/123", ... } ``` @@ -338,7 +338,6 @@ see the `group_saml` option: "username": "john_smith", "shared_runners_minutes_limit": 133, "extra_shared_runners_minutes_limit": 133, - "note": "DMCA Request: 2018-11-05 | DMCA Violation | Abuse | https://gitlab.zendesk.com/agent/tickets/123", "identities": [ {"provider": "github", "extern_uid": "2435223452345"}, {"provider": "bitbucket", "extern_uid": "john.smith"}, @@ -376,7 +375,7 @@ POST /users Parameters: | Attribute | Required | Description | -|:-------------------------------------|:---------|:--------------------------------------------------------------------------------------------------------------------------------------------------------| +| :----------------------------------- | :------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | | `admin` | No | User is admin - true or false (default) | | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | @@ -391,6 +390,7 @@ Parameters: | `linkedin` | No | LinkedIn | | `location` | No | User's location | | `name` | Yes | Name | +| `note` | No | Admin notes for this user | | `organization` | No | Organization name | | `password` | No | Password | | `private_profile` | No | User's profile is private - true, false (default), or null (will be converted to false) | @@ -417,7 +417,7 @@ PUT /users/:id Parameters: | Attribute | Required | Description | -|:-------------------------------------|:---------|:--------------------------------------------------------------------------------------------------------------------------------------------------------| +| :----------------------------------- | :------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | | `admin` | No | User is admin - true or false (default) | | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | @@ -432,7 +432,7 @@ Parameters: | `linkedin` | No | LinkedIn | | `location` | No | User's location | | `name` | No | Name | -| `note` | No | Admin notes for this user **(STARTER)** | +| `note` | No | Admin notes for this user | | `organization` | No | Organization name | | `password` | No | Password | | `private_profile` | No | User's profile is private - true, false (default), or null (will be converted to false) | @@ -609,8 +609,8 @@ Get the status of a user. GET /users/:id_or_username/status ``` -| Attribute | Type | Required | Description | -| ---------------- | ------ | -------- | ----------- | +| Attribute | Type | Required | Description | +| ---------------- | ------ | -------- | ------------------------------------------------- | | `id_or_username` | string | yes | The ID or username of the user to get a status of | ```shell @@ -635,15 +635,15 @@ Set the status of the current user. PUT /user/status ``` -| Attribute | Type | Required | Description | -| --------- | ------ | -------- | ----------- | -| `emoji` | string | no | The name of the emoji to use as status, if omitted `speech_balloon` is used. Emoji name can be one of the specified names in the [Gemojione index](https://github.com/bonusly/gemojione/blob/master/config/index.json). | -| `message` | string | no | The message to set as a status. It can also contain emoji codes. | +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `emoji` | string | no | The name of the emoji to use as status. If omitted `speech_balloon` is used. Emoji name can be one of the specified names in the [Gemojione index](https://github.com/bonusly/gemojione/blob/master/config/index.json). | +| `message` | string | no | The message to set as a status. It can also contain emoji codes. | When both parameters `emoji` and `message` are empty, the status will be cleared. ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "emoji=coffee" --data "message=I crave coffee" https://gitlab.example.com/api/v4/user/status +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "emoji=coffee" --data "message=I crave coffee" "https://gitlab.example.com/api/v4/user/status" ``` Example responses @@ -660,9 +660,9 @@ Example responses Get the counts (same as in top right menu) of the currently signed in user. -| Attribute | Type | Description | -| --------- | ---- | ----------- | -| `merge_requests` | number | Merge requests that are active and assigned to current user. | +| Attribute | Type | Description | +| ---------------- | ------ | ------------------------------------------------------------ | +| `merge_requests` | number | Merge requests that are active and assigned to current user. | ```plaintext GET /user_counts @@ -721,8 +721,8 @@ Get a list of a specified user's SSH keys. GET /users/:id_or_username/keys ``` -| Attribute | Type | Required | Description | -| ---------------- | ------ | -------- | ----------- | +| Attribute | Type | Required | Description | +| ---------------- | ------ | -------- | ------------------------------------------------------- | | `id_or_username` | string | yes | The ID or username of the user to get the SSH keys for. | ## Single SSH key @@ -758,13 +758,13 @@ Parameters: - `title` (required) - new SSH Key's title - `key` (required) - new SSH key +- `expires_at` (optional) - The expiration date of the SSH key in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`) ```json { - "created_at": "2015-01-21T17:44:33.512Z", - "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAMLrhYgI3atfrSD6KDas1b/3n6R/HP+bLaHHX6oh+L1vg31mdUqK0Ac/NjZoQunavoyzqdPYhFz9zzOezCrZKjuJDS3NRK9rspvjgM0xYR4d47oNZbdZbwkI4cTv/gcMlquRy0OvpfIvJtjtaJWMwTLtM5VhRusRuUlpH99UUVeXAAAAFQCVyX+92hBEjInEKL0v13c/egDCTQAAAIEAvFdWGq0ccOPbw4f/F8LpZqvWDydAcpXHV3thwb7WkFfppvm4SZte0zds1FJ+Hr8Xzzc5zMHe6J4Nlay/rP4ewmIW7iFKNBEYb/yWa+ceLrs+TfR672TaAgO6o7iSRofEq5YLdwgrwkMmIawa21FrZ2D9SPao/IwvENzk/xcHu7YAAACAQFXQH6HQnxOrw4dqf0NqeKy1tfIPxYYUZhPJfo9O0AmBW2S36pD2l14kS89fvz6Y1g8gN/FwFnRncMzlLY/hX70FSc/3hKBSbH6C6j8hwlgFKfizav21eS358JJz93leOakJZnGb8XlWvz1UJbwCsnR2VEY8Dz90uIk1l/UqHkA= loic@call", "title": "ABC", - "id": 4 + "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAMLrhYgI3atfrSD6KDas1b/3n6R/HP+bLaHHX6oh+L1vg31mdUqK0Ac/NjZoQunavoyzqdPYhFz9zzOezCrZKjuJDS3NRK9rspvjgM0xYR4d47oNZbdZbwkI4cTv/gcMlquRy0OvpfIvJtjtaJWMwTLtM5VhRusRuUlpH99UUVeXAAAAFQCVyX+92hBEjInEKL0v13c/egDCTQAAAIEAvFdWGq0ccOPbw4f/F8LpZqvWDydAcpXHV3thwb7WkFfppvm4SZte0zds1FJ+Hr8Xzzc5zMHe6J4Nlay/rP4ewmIW7iFKNBEYb/yWa+ceLrs+TfR672TaAgO6o7iSRofEq5YLdwgrwkMmIawa21FrZ2D9SPao/IwvENzk/xcHu7YAAACAQFXQH6HQnxOrw4dqf0NqeKy1tfIPxYYUZhPJfo9O0AmBW2S36pD2l14kS89fvz6Y1g8gN/FwFnRncMzlLY/hX70FSc/3hKBSbH6C6j8hwlgFKfizav21eS358JJz93leOakJZnGb8XlWvz1UJbwCsnR2VEY8Dz90uIk1l/UqHkA= loic@call", + "expires_at": "2016-01-21T00:00:00.000Z" } ``` @@ -797,6 +797,7 @@ Parameters: - `id` (required) - ID of specified user - `title` (required) - new SSH Key's title - `key` (required) - new SSH key +- `expires_at` (optional) - The expiration date of the SSH key in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`) ## Delete SSH key for current user @@ -833,7 +834,7 @@ GET /user/gpg_keys ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/user/gpg_keys +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/user/gpg_keys" ``` Example response: @@ -858,12 +859,12 @@ GET /user/gpg_keys/:key_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | ----------- | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | | `key_id` | integer | yes | The ID of the GPG key | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/user/gpg_keys/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/user/gpg_keys/1" ``` Example response: @@ -886,12 +887,12 @@ POST /user/gpg_keys Parameters: -| Attribute | Type | Required | Description | -| --------- | ------ | -------- | ----------- | +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | --------------- | | key | string | yes | The new GPG key | ```shell -curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/user/gpg_keys +curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/user/gpg_keys" ``` Example response: @@ -916,12 +917,12 @@ DELETE /user/gpg_keys/:key_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | ----------- | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | | `key_id` | integer | yes | The ID of the GPG key | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/user/gpg_keys/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/user/gpg_keys/1" ``` Returns `204 No Content` on success, or `404 Not found` if the key cannot be found. @@ -936,12 +937,12 @@ GET /users/:id/gpg_keys Parameters: -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | ----------- | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | ------------------ | | `id` | integer | yes | The ID of the user | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/2/gpg_keys +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/2/gpg_keys" ``` Example response: @@ -966,13 +967,13 @@ GET /users/:id/gpg_keys/:key_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | ----------- | -| `id` | integer | yes | The ID of the user | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | +| `id` | integer | yes | The ID of the user | | `key_id` | integer | yes | The ID of the GPG key | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/2/gpg_keys/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/2/gpg_keys/1" ``` Example response: @@ -995,13 +996,13 @@ POST /users/:id/gpg_keys Parameters: -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | ----------- | -| `id` | integer | yes | The ID of the user | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | +| `id` | integer | yes | The ID of the user | | `key_id` | integer | yes | The ID of the GPG key | ```shell -curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/2/gpg_keys +curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/2/gpg_keys" ``` Example response: @@ -1026,13 +1027,13 @@ DELETE /users/:id/gpg_keys/:key_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | ----------- | -| `id` | integer | yes | The ID of the user | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | --------------------- | +| `id` | integer | yes | The ID of the user | | `key_id` | integer | yes | The ID of the GPG key | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/2/gpg_keys/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/2/gpg_keys/1" ``` ## List emails @@ -1198,7 +1199,7 @@ Will return `201 OK` on success, `404 User Not Found` is user cannot be found or ## Deactivate user -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/22257) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22257) in GitLab 12.4. Deactivates the specified user. Available only for admin. @@ -1220,7 +1221,7 @@ Returns: ## Activate user -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/22257) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22257) in GitLab 12.4. Activates the specified user. Available only for admin. @@ -1261,7 +1262,7 @@ Parameters: | `state` | string | no | filter tokens based on state (`all`, `active`, `inactive`) | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/42/impersonation_tokens +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/42/impersonation_tokens" ``` Example response: @@ -1313,7 +1314,7 @@ Parameters: | `impersonation_token_id` | integer | yes | The ID of the impersonation token | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/42/impersonation_tokens/2 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/42/impersonation_tokens/2" ``` Example response: @@ -1347,15 +1348,15 @@ settings page. POST /users/:user_id/impersonation_tokens ``` -| Attribute | Type | Required | Description | -| ------------ | ------- | -------- | ----------- | -| `user_id` | integer | yes | The ID of the user | -| `name` | string | yes | The name of the impersonation token | -| `expires_at` | date | no | The expiration date of the impersonation token in ISO format (`YYYY-MM-DD`)| -| `scopes` | array | yes | The array of scopes of the impersonation token (`api`, `read_user`) | +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | --------------------------------------------------------------------------- | +| `user_id` | integer | yes | The ID of the user | +| `name` | string | yes | The name of the impersonation token | +| `expires_at` | date | no | The expiration date of the impersonation token in ISO format (`YYYY-MM-DD`) | +| `scopes` | array | yes | The array of scopes of the impersonation token (`api`, `read_user`) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "name=mytoken" --data "expires_at=2017-04-04" --data "scopes[]=api" https://gitlab.example.com/api/v4/users/42/impersonation_tokens +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --data "name=mytoken" --data "expires_at=2017-04-04" --data "scopes[]=api" "https://gitlab.example.com/api/v4/users/42/impersonation_tokens" ``` Example response: @@ -1387,15 +1388,15 @@ DELETE /users/:user_id/impersonation_tokens/:impersonation_token_id ``` ```shell -curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/42/impersonation_tokens/1 +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/42/impersonation_tokens/1" ``` Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `user_id` | integer | yes | The ID of the user | -| `impersonation_token_id` | integer | yes | The ID of the impersonation token | +| Attribute | Type | Required | Description | +| ------------------------ | ------- | -------- | --------------------------------- | +| `user_id` | integer | yes | The ID of the user | +| `impersonation_token_id` | integer | yes | The ID of the impersonation token | ### Get user activities (admin only) @@ -1406,8 +1407,8 @@ Get the last activity date for all users, sorted from oldest to newest. The activities that update the timestamp are: - Git HTTP/SSH activities (such as clone, push) -- User logging in into GitLab -- User visiting pages related to Dashboards, Projects, Issues, and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/54947) in GitLab 11.8) +- User logging in to GitLab +- User visiting pages related to Dashboards, Projects, Issues, and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54947) in GitLab 11.8) - User using the API - User using the GraphQL API @@ -1420,12 +1421,12 @@ GET /user/activities Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `from` | string | no | Date string in the format YEAR-MONTH-DAY. For example, `2016-03-11`. Defaults to 6 months ago. | +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | ---------------------------------------------------------------------------------------------- | +| `from` | string | no | Date string in the format YEAR-MONTH-DAY. For example, `2016-03-11`. Defaults to 6 months ago. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/user/activities +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/user/activities" ``` Example response: @@ -1454,7 +1455,7 @@ Please note that `last_activity_at` is deprecated, please use `last_activity_on` ## User memberships (admin only) -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/20532) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20532) in GitLab 12.8. Lists all projects and groups a user is a member of. This endpoint is available for admins only. It returns the `source_id`, `source_name`, `source_type` and `access_level` of a membership. @@ -1467,10 +1468,10 @@ GET /users/:id/memberships Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of a specified user | -| `type` | string | no | Filter memberships by type. Can be either `Project` or `Namespace` | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | ------------------------------------------------------------------ | +| `id` | integer | yes | The ID of a specified user | +| `type` | string | no | Filter memberships by type. Can be either `Project` or `Namespace` | Returns: @@ -1480,7 +1481,7 @@ Returns: - `400 Bad Request` when requested type is not supported. ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/users/<user_id>/memberships +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/:user_id/memberships" ``` Example response: diff --git a/doc/api/v3_to_v4.md b/doc/api/v3_to_v4.md index 76e5dd9abb6..4571d4d8304 100644 --- a/doc/api/v3_to_v4.md +++ b/doc/api/v3_to_v4.md @@ -3,7 +3,7 @@ Since GitLab 9.0, API V4 is the preferred version to be used. API V3 was unsupported from GitLab 9.5, released on August -22, 2017. API v3 was removed in [GitLab 11.0](https://gitlab.com/gitlab-org/gitlab-foss/issues/36819). +22, 2017. API v3 was removed in [GitLab 11.0](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/36819). The V3 API documentation is still [available](https://gitlab.com/gitlab-org/gitlab-foss/blob/8-16-stable/doc/api/README.md). diff --git a/doc/api/version.md b/doc/api/version.md index 6c9ff6ac9e1..661fdfb4b03 100644 --- a/doc/api/version.md +++ b/doc/api/version.md @@ -10,7 +10,7 @@ GET /version ``` ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/version +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/version" ``` Example response: diff --git a/doc/api/visual_review_discussions.md b/doc/api/visual_review_discussions.md index 161f84f4618..bcbfdbdc6d0 100644 --- a/doc/api/visual_review_discussions.md +++ b/doc/api/visual_review_discussions.md @@ -36,5 +36,5 @@ Parameters: | `position[y]` | integer | no | Y coordinate (Only stored for `image` diff notes) | ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/merge_requests/11/visual_review_discussions?body=comment +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/visual_review_discussions?body=comment" ``` diff --git a/doc/api/vulnerabilities.md b/doc/api/vulnerabilities.md index ff1a6a7ebcd..70f29d961e3 100644 --- a/doc/api/vulnerabilities.md +++ b/doc/api/vulnerabilities.md @@ -1,6 +1,6 @@ # Vulnerabilities API **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10242) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10242) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. NOTE: **Note:** The former Vulnerabilities API was renamed to Vulnerability Findings API @@ -9,14 +9,6 @@ This document now describes the new Vulnerabilities API that provides access to [Standalone Vulnerabilities](https://gitlab.com/groups/gitlab-org/-/epics/634). CAUTION: **Caution:** -This API is currently in development and is protected by a **disabled** -[feature flag](../development/feature_flags/index.md). -On a self-managed GitLab instance, an administrator can enable it by starting the Rails console -(`sudo gitlab-rails console`) and then running the following command: `Feature.enable(:first_class_vulnerabilities)`. -To test if the Vulnerabilities API was successfully enabled, run the following command: -`Feature.enabled?(:first_class_vulnerabilities)`. - -CAUTION: **Caution:** This API is in an alpha stage and considered unstable. The response payload may be subject to change or breakage across GitLab releases. @@ -40,7 +32,7 @@ GET /vulnerabilities/:id | `id` | integer or string | yes | The ID of a Vulnerability to get | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/vulnerabilities/1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/vulnerabilities/1" ``` Example response: diff --git a/doc/api/vulnerability_exports.md b/doc/api/vulnerability_exports.md index 2c9ac5d65eb..2cb647e797b 100644 --- a/doc/api/vulnerability_exports.md +++ b/doc/api/vulnerability_exports.md @@ -1,14 +1,6 @@ # Vulnerability export API **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197494) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. [Updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30397) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. - -CAUTION: **Caution:** -This API is currently in development and is protected by a **disabled** -[feature flag](../development/feature_flags/index.md). -On a self-managed GitLab instance, an administrator can enable it by starting the Rails console -(`sudo gitlab-rails console`) and then running the following command: `Feature.enable(:first_class_vulnerabilities)`. -To test if the Vulnerability Exports API was successfully enabled, run the following command: -`Feature.enabled?(:first_class_vulnerabilities)`. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/197494) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. [Updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30397) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. CAUTION: **Caution:** This API is in an alpha stage and considered unstable. @@ -39,7 +31,7 @@ POST /security/projects/:id/vulnerability_exports | `id` | integer or string | yes | The ID or [URL-encoded path](README.md#namespaced-path-encoding) of the project which the authenticated user is a member of | ```shell -curl --header POST "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/projects/1/vulnerability_exports +curl --header POST "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/vulnerability_exports" ``` The created vulnerability export is automatically deleted after 1 hour. @@ -155,7 +147,7 @@ GET /security/vulnerability_exports/:id | `id` | integer or string | yes | The vulnerability export's ID | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/vulnerability_exports/2 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/security/vulnerability_exports/2" ``` If the vulnerability export isn't finished, the response is `202 Accepted`. @@ -192,7 +184,7 @@ GET /security/vulnerability_exports/:id/download | `id` | integer or string | yes | The vulnerability export's ID | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/security/vulnerability_exports/2/download +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/security/vulnerability_exports/2/download" ``` The response will be `404 Not Found` if the vulnerability export is not finished yet or was not found. diff --git a/doc/api/vulnerability_findings.md b/doc/api/vulnerability_findings.md index f5e607d6829..7fbd58ea62c 100644 --- a/doc/api/vulnerability_findings.md +++ b/doc/api/vulnerability_findings.md @@ -4,7 +4,7 @@ NOTE: **Note:** This API resource is renamed from Vulnerabilities to Vulnerability Findings because the Vulnerabilities are reserved -for serving the upcoming [Standalone Vulnerability objects](https://gitlab.com/gitlab-org/gitlab/issues/13561). +for serving the upcoming [Standalone Vulnerability objects](https://gitlab.com/gitlab-org/gitlab/-/issues/13561). To fix any broken integrations with the former Vulnerabilities API, change the `vulnerabilities` URL part to be `vulnerability_findings`. @@ -47,19 +47,19 @@ GET /projects/:id/vulnerability_findings?pipeline_id=42 ``` CAUTION: **Deprecation:** -Beginning with GitLab 12.9, the `undefined` severity level is deprecated and the `undefined` confidence level isn't reported for new vulnerabilities. +Beginning with GitLab 12.9, the `undefined` severity and confidence level is no longer reported. | Attribute | Type | Required | Description | | ------------- | -------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) which the authenticated user is a member of. | | `report_type` | string array | no | Returns vulnerability findings belonging to specified report type. Valid values: `sast`, `dast`, `dependency_scanning`, or `container_scanning`. Defaults to all. | | `scope` | string | no | Returns vulnerability findings for the given scope: `all` or `dismissed`. Defaults to `dismissed`. | -| `severity` | string array | no | Returns vulnerability findings belonging to specified severity level: `undefined`, `info`, `unknown`, `low`, `medium`, `high`, or `critical`. Defaults to all. | -| `confidence` | string array | no | Returns vulnerability findings belonging to specified confidence level: `undefined`, `ignore`, `unknown`, `experimental`, `low`, `medium`, `high`, or `confirmed`. Defaults to all. | +| `severity` | string array | no | Returns vulnerability findings belonging to specified severity level: `info`, `unknown`, `low`, `medium`, `high`, or `critical`. Defaults to all. | +| `confidence` | string array | no | Returns vulnerability findings belonging to specified confidence level: `ignore`, `unknown`, `experimental`, `low`, `medium`, `high`, or `confirmed`. Defaults to all. | | `pipeline_id` | integer/string | no | Returns vulnerability findings belonging to specified pipeline. | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/vulnerability_findings +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/vulnerability_findings" ``` Example response: diff --git a/doc/api/wikis.md b/doc/api/wikis.md index 48b04fefd39..f7595e7efe6 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -18,7 +18,7 @@ GET /projects/:id/wikis | `with_content` | boolean | no | Include pages' content | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/wikis?with_content=1 +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/wikis?with_content=1" ``` Example response: @@ -59,7 +59,7 @@ GET /projects/:id/wikis/:slug | `slug` | string | yes | The slug (a unique string) of the wiki page | ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/1/wikis/home +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/wikis/home" ``` Example response: @@ -174,7 +174,7 @@ The `file=` parameter must point to a file on your filesystem and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" https://gitlab.example.com/api/v4/projects/1/wikis/attachments +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/1/wikis/attachments" ``` Example response: diff --git a/doc/articles/how_to_configure_ldap_gitlab_ce/index.md b/doc/articles/how_to_configure_ldap_gitlab_ce/index.md index 8e2e54711e7..2fbeb6f2506 100644 --- a/doc/articles/how_to_configure_ldap_gitlab_ce/index.md +++ b/doc/articles/how_to_configure_ldap_gitlab_ce/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../administration/auth/how_to_configure_ldap_gitlab_ce/index.md' +redirect_to: '../../administration/auth/ldap/index.md' --- -This document was moved to [another location](../../administration/auth/how_to_configure_ldap_gitlab_ce/index.md). +This document was moved to [another location](../../administration/auth/ldap/index.md). diff --git a/doc/articles/how_to_configure_ldap_gitlab_ee/index.md b/doc/articles/how_to_configure_ldap_gitlab_ee/index.md index 3e6f3130437..2fbeb6f2506 100644 --- a/doc/articles/how_to_configure_ldap_gitlab_ee/index.md +++ b/doc/articles/how_to_configure_ldap_gitlab_ee/index.md @@ -1,5 +1,5 @@ --- -redirect_to: '../../administration/auth/how_to_configure_ldap_gitlab_ee/index.md' +redirect_to: '../../administration/auth/ldap/index.md' --- -This document was moved to [another location](../../administration/auth/how_to_configure_ldap_gitlab_ee/index.md). +This document was moved to [another location](../../administration/auth/ldap/index.md). diff --git a/doc/articles/index.md b/doc/articles/index.md index 0cd9baba20e..7fc4c18f771 100644 --- a/doc/articles/index.md +++ b/doc/articles/index.md @@ -8,7 +8,7 @@ Technical articles are topic-related documentation, written with a user-friendly approach and language, aiming to provide the community with guidance on specific processes to achieve certain objectives. -The list of technical articles was [deprecated](https://gitlab.com/gitlab-org/gitlab-foss/issues/41138) in favor of having them linked from their topic-related documentation: +The list of technical articles was [deprecated](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41138) in favor of having them linked from their topic-related documentation: - [Git](../topics/git/index.md) - [GitLab administrator](../administration/index.md) diff --git a/doc/ci/README.md b/doc/ci/README.md index fce0ad15b70..150f160b762 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false description: "Learn how to use GitLab CI/CD, the GitLab built-in Continuous Integration, Continuous Deployment, and Continuous Delivery toolset to build, test, and deploy your application." type: index @@ -57,8 +60,10 @@ the following documents: - [GitLab CI/CD basic workflow](introduction/index.md#basic-cicd-workflow). - [Step-by-step guide for writing `.gitlab-ci.yml` for the first time](../user/project/pages/getting_started_part_four.md). -If you're coming over from Jenkins, you can also check out our handy [reference](jenkins/index.md) -for converting your pipelines. +If you're migrating from another CI/CD tool, check out our handy references: + +- [Migrating from CircleCI](migration/circleci.md) +- [Migrating from Jenkins](jenkins/index.md) You can also get started by using one of the [`.gitlab-ci.yml` templates](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/lib/gitlab/ci/templates) @@ -66,7 +71,7 @@ available through the UI. You can use them by creating a new file, choosing a template that suits your application, and adjusting it to your needs: - + For a broader overview, see the [CI/CD getting started](quick_start/README.md) guide. @@ -75,7 +80,7 @@ Once you're familiar with how GitLab CI/CD works, see the for all the attributes you can set and use. NOTE: **Note:** -GitLab CI/CD and [shared runners](runners/README.md#shared-specific-and-group-runners) are enabled in GitLab.com and available for all users, limited only to the [user's pipelines quota](../user/gitlab_com/index.md#shared-runners). +GitLab CI/CD and [shared runners](runners/README.md#shared-runners) are enabled in GitLab.com and available for all users, limited only to the [user's pipelines quota](../user/gitlab_com/index.md#shared-runners). ## Concepts @@ -103,7 +108,7 @@ GitLab CI/CD supports numerous configuration options: | [Pipeline triggers](triggers/README.md) | Trigger pipelines through the API. | | [Pipelines for Merge Requests](merge_request_pipelines/index.md) | Design a pipeline structure for running a pipeline in merge requests. | | [Integrate with Kubernetes clusters](../user/project/clusters/index.md) | Connect your project to Google Kubernetes Engine (GKE) or an existing Kubernetes cluster. | -| [Optimize GitLab and Runner for large repositories](large_repositories/index.md) | Recommended strategies for handling large repos. | +| [Optimize GitLab and Runner for large repositories](large_repositories/index.md) | Recommended strategies for handling large repositories. | | [`.gitlab-ci.yml` full reference](yaml/README.md) | All the attributes you can use with GitLab CI/CD. | Note that certain operations can only be performed according to the @@ -183,22 +188,43 @@ See also the [Why CI/CD?](https://docs.google.com/presentation/d/1OGgk2Tcxbpl7DJ As GitLab CI/CD has evolved, certain breaking changes have been necessary. These are: +#### 13.0 + +- [Remove Backported + `os.Expand`](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4915) +- [Remove Fedora 29 package + support](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/16158) +- [Remove macOS 32-bit + support](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/25466) +- [Removed `debug/jobs/list?v=1` + endpoint](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6361) +- [Remove support for array of strings when defining services for Docker + executor](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4922) +- [Remove `--docker-services` flag on register + command](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6404) +- [Remove legacy build directory + caching](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4180) +- [Remove `FF_USE_LEGACY_VOLUMES_MOUNTING_ORDER` feature + flag](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6581) +- [Remove support for Windows Server + 1803](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6553) + #### 12.0 - [Use refspec to clone/fetch Git - repository](https://gitlab.com/gitlab-org/gitlab-runner/issues/4069). + repository](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4069). - [Old cache - configuration](https://gitlab.com/gitlab-org/gitlab-runner/issues/4070). + configuration](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4070). - [Old metrics server - configuration](https://gitlab.com/gitlab-org/gitlab-runner/issues/4072). + configuration](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4072). - [Remove - `FF_K8S_USE_ENTRYPOINT_OVER_COMMAND`](https://gitlab.com/gitlab-org/gitlab-runner/issues/4073). + `FF_K8S_USE_ENTRYPOINT_OVER_COMMAND`](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4073). - [Remove Linux distributions that reach EOL](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1130). - [Update command line API for helper - images](https://gitlab.com/gitlab-org/gitlab-runner/issues/4013). + images](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4013). - [Remove old `git clean` - flow](https://gitlab.com/gitlab-org/gitlab-runner/issues/4175). + flow](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4175). #### 11.0 diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index 16cabae353e..392dd4fdeba 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index, concepts, howto --- @@ -83,9 +86,9 @@ which type of Runner you are using, cache can act differently. From the perspective of the developer, to ensure maximum availability of the cache, when declaring `cache` in your jobs, use one or a mix of the following: -- [Tag your Runners](../runners/README.md#using-tags) and use the tag on jobs +- [Tag your Runners](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) and use the tag on jobs that share their cache. -- [Use sticky Runners](../runners/README.md#locking-a-specific-runner-from-being-enabled-for-other-projects) +- [Use sticky Runners](../runners/README.md#prevent-a-specific-runner-from-being-enabled-for-other-projects) that will be only available to a particular project. - [Use a `key`](../yaml/README.md#cachekey) that fits your workflow (for example, different caches on each branch). For that, you can take advantage of the @@ -556,7 +559,7 @@ next run of the pipeline, the cache will be stored in a different location. ### Clearing the cache manually -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41249) in GitLab 10.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41249) in GitLab 10.4. If you want to avoid editing `.gitlab-ci.yml`, you can easily clear the cache via GitLab's UI: diff --git a/doc/ci/chatops/README.md b/doc/ci/chatops/README.md index 5488834a24e..a8fea0836c1 100644 --- a/doc/ci/chatops/README.md +++ b/doc/ci/chatops/README.md @@ -1,4 +1,7 @@ --- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index, concepts, howto --- diff --git a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md index 2836f9932c6..ba801950c40 100644 --- a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md +++ b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- diff --git a/doc/ci/ci_cd_for_external_repos/github_integration.md b/doc/ci/ci_cd_for_external_repos/github_integration.md index 399205d66cf..5365ef49944 100644 --- a/doc/ci/ci_cd_for_external_repos/github_integration.md +++ b/doc/ci/ci_cd_for_external_repos/github_integration.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -12,7 +15,7 @@ GitLab. Watch a video on [Using GitLab CI/CD pipelines with GitHub repositories](https://www.youtube.com/watch?v=qgl3F2j-1cI). NOTE: **Note:** -Because of [GitHub limitations](https://gitlab.com/gitlab-org/gitlab/issues/9147), +Because of [GitHub limitations](https://gitlab.com/gitlab-org/gitlab/-/issues/9147), [GitHub OAuth](../../integration/github.md#enabling-github-oauth) cannot be used to authenticate with GitHub as an external CI/CD repository. @@ -20,7 +23,7 @@ cannot be used to authenticate with GitHub as an external CI/CD repository. NOTE: **Note:** Personal access tokens can only be used to connect GitHub.com -repositories to GitLab. +repositories to GitLab, and the GitHub user must have the [owner role](https://help.github.com/en/github/getting-started-with-github/access-permissions-on-github). To perform a one-off authorization with GitHub to grant GitLab access your repositories: @@ -33,11 +36,9 @@ repositories: your project, update commit statuses, and create a web hook to notify GitLab of new commits. -1. In GitLab create a **CI/CD for external repo** project and select +1. In GitLab, go to the [new project page](../../gitlab-basics/create-project.md#create-a-project-in-gitlab), select the **CI/CD for external repo** tab, and then click **GitHub**. -  - 1. Paste the token into the **Personal access token** field and click **List Repositories**. Click **Connect** to select the repository. @@ -91,7 +92,7 @@ To manually enable GitLab CI/CD for your repository: https://gitlab.com/api/v4/projects/<NAMESPACE>%2F<PROJECT>/mirror/pull?private_token=<PERSONAL_ACCESS_TOKEN> ``` -  + Select the **Let me select individual events** option, then check the **Pull requests** and **Pushes** checkboxes. These settings are needed for [pipelines for external pull requests](index.md#pipelines-for-external-pull-requests). 1. In GitHub add a `.gitlab-ci.yml` to configure GitLab CI/CD. diff --git a/doc/ci/ci_cd_for_external_repos/index.md b/doc/ci/ci_cd_for_external_repos/index.md index cc6645b72c2..78988e8a057 100644 --- a/doc/ci/ci_cd_for_external_repos/index.md +++ b/doc/ci/ci_cd_for_external_repos/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: index, howto --- @@ -31,7 +34,7 @@ To connect to an external repository: ## Pipelines for external pull requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/65139) in GitLab Premium 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/65139) in GitLab Premium 12.3. When using GitLab CI/CD with an [external repository on GitHub](github_integration.md), it's possible to run a pipeline in the context of a Pull Request. @@ -87,10 +90,10 @@ The variable names are prefixed with `CI_EXTERNAL_PULL_REQUEST_`. ### Limitations -This feature currently does not support Pull Requests from fork repositories. Any Pull Requests from fork repositories will be ignored. [Read more](https://gitlab.com/gitlab-org/gitlab/issues/5667). +This feature currently does not support Pull Requests from fork repositories. Any Pull Requests from fork repositories will be ignored. [Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/5667). Given that GitLab will create 2 pipelines, if changes are pushed to a remote branch that references an open Pull Request, both will contribute to the status of the Pull Request via GitHub integration. If you want to exclusively run pipelines on external pull requests and not on branches you can add `except: [branches]` to the job specs. -[Read more](https://gitlab.com/gitlab-org/gitlab/issues/24089#workaround). +[Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/24089#workaround). diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index 8883ad5dfd6..26eb99bd55a 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -7,7 +7,7 @@ type: howto # Cloud deployment -Interacting with a major cloud provider such as Amazon AWS may have become a much needed task that's +Interacting with a major cloud provider may have become a much needed task that's part of your delivery process. GitLab is making this process less painful by providing Docker images that come with the needed libraries and tools pre-installed. By referencing them in your CI/CD pipeline, you'll be able to interact with your chosen @@ -15,7 +15,12 @@ cloud provider more easily. ## AWS -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31167) in GitLab 12.6. +GitLab provides Docker images to simplify working with AWS, and a template to make +it easier to [deploy to AWS](#deploy-your-application-to-the-aws-elastic-container-service-ecs). + +### Run AWS commands from GitLab CI/CD + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31167) in GitLab 12.6. GitLab's AWS Docker image provides the [AWS Command Line Interface](https://aws.amazon.com/cli/), which enables you to run `aws` commands. As part of your deployment strategy, you can run `aws` commands directly from @@ -30,12 +35,21 @@ Some credentials are required to be able to run `aws` commands: NOTE: **Note:** A new **Access key ID** and **Secret access key** pair will be generated. Please take a note of them right away. -1. In your GitLab project, go to **Settings > CI / CD**. Set the Access key ID and Secret access key as [environment variables](../variables/README.md#gitlab-cicd-environment-variables), using the following variable names: +1. In your GitLab project, go to **Settings > CI / CD**. Set the following as + [environment variables](../variables/README.md#gitlab-cicd-environment-variables) + (see table below): - | Env. variable name | Value | - |:------------------------|:-------------------------| - | `AWS_ACCESS_KEY_ID` | Your "Access key ID" | - | `AWS_SECRET_ACCESS_KEY` | Your "Secret access key" | + - Access key ID. + - Secret access key. + - Region code. You can check the [list of AWS regional endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). + You might want to check if the AWS service you intend to use is + [available in the chosen region](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/). + + | Env. variable name | Value | + |:------------------------|:-----------------------| + | `AWS_ACCESS_KEY_ID` | Your Access key ID | + | `AWS_SECRET_ACCESS_KEY` | Your Secret access key | + | `AWS_DEFAULT_REGION` | Your region code | 1. You can now use `aws` commands in the `.gitlab-ci.yml` file of this project: @@ -49,25 +63,25 @@ Some credentials are required to be able to run `aws` commands: ``` NOTE: **Note:** - Please note that the image used in the example above + The image used in the example above (`registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest`) is hosted on the [GitLab Container Registry](../../user/packages/container_registry/index.md) and is - ready to use. Alternatively, replace the image with another one hosted on [AWS ECR](#aws-ecr). + ready to use. Alternatively, replace the image with one hosted on AWS ECR. -### AWS ECR +### Use an AWS Elastic Container Registry (ECR) image in your CI/CD -Instead of referencing an image hosted on the GitLab Registry, you are free to -reference any other image hosted on any third-party registry, such as +Instead of referencing an image hosted on the GitLab Registry, you can +reference an image hosted on any third-party registry, such as the [Amazon Elastic Container Registry (ECR)](https://aws.amazon.com/ecr/). -To do so, please make sure to [push your image into your ECR -repository](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html) -before referencing it in your `.gitlab-ci.yml` file and replace the `image` -path to point to your ECR. +To do so, [push your image into your ECR +repository](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html). +Then reference it in your `.gitlab-ci.yml` file and replace the `image` +path to point to your ECR image. -### Deploy your application to AWS Elastic Container Service (ECS) +### Deploy your application to the AWS Elastic Container Service (ECS) -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/207962) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207962) in GitLab 12.9. GitLab provides a series of [CI templates that you can include in your project](../yaml/README.md#include). To automate deployments of your application to your [Amazon Elastic Container Service](https://aws.amazon.com/ecs/) (AWS ECS) @@ -80,7 +94,7 @@ components, like an ECS service, ECS task definition, a database on AWS RDS, etc After you're all set up on AWS ECS, follow these steps: 1. Make sure your AWS credentials are set up as environment variables for your - project. You can follow [the steps above](#aws) to complete this setup. + project. You can follow [the steps above](#run-aws-commands-from-gitlab-cicd) to complete this setup. 1. Add these variables to your project's `.gitlab-ci.yml` file: ```yaml @@ -107,11 +121,11 @@ After you're all set up on AWS ECS, follow these steps: ```yaml include: - - template: Deploy-ECS.gitlab-ci.yml + - template: AWS/Deploy-ECS.gitlab-ci.yml ``` The `Deploy-ECS` template ships with GitLab and is available [on - GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml). + GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml). 1. Commit and push your updated `.gitlab-ci.yml` to your project's repository, and you're done! @@ -123,6 +137,17 @@ After you're all set up on AWS ECS, follow these steps: task definition, making the cluster pull the newest version of your application. +CAUTION: **Warning:** +The [`Deploy-ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml) +template includes both the [`Jobs/Build.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml) +and [`Jobs/Deploy/ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml) +"sub-templates". Do not include these "sub-templates" on their own, and only include the main +`Deploy-ECS.gitlab-ci.yml` template. The "sub-templates" are designed to only be +used along with the main template. They may move or change unexpectedly causing your +pipeline to fail if you didn't include the main template. Also, the job names within +these templates may change. Do not override these jobs names in your own pipeline, +as the override will stop working when the name changes. + Alternatively, if you don't wish to use the `Deploy-ECS.gitlab-ci.yml` template to deploy to AWS ECS, you can always use our `aws-base` Docker image to run your own [AWS CLI commands for ECS](https://docs.aws.amazon.com/cli/latest/reference/ecs/index.html#cli-aws-ecs). diff --git a/doc/ci/directed_acyclic_graph/img/dag_graph_example_clicked_v13_1.png b/doc/ci/directed_acyclic_graph/img/dag_graph_example_clicked_v13_1.png Binary files differnew file mode 100644 index 00000000000..3610d471ef4 --- /dev/null +++ b/doc/ci/directed_acyclic_graph/img/dag_graph_example_clicked_v13_1.png diff --git a/doc/ci/directed_acyclic_graph/img/dag_graph_example_v13_1.png b/doc/ci/directed_acyclic_graph/img/dag_graph_example_v13_1.png Binary files differnew file mode 100644 index 00000000000..bd5215d31c8 --- /dev/null +++ b/doc/ci/directed_acyclic_graph/img/dag_graph_example_v13_1.png diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index 8722efd3b40..fff0fda0ab4 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -1,10 +1,13 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Directed Acyclic Graph -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47063) in GitLab 12.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) in GitLab 12.2. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/206902) in GitLab 12.10. A [directed acyclic graph](https://www.techopedia.com/definition/5739/directed-acyclic-graph-dag) can be @@ -75,3 +78,34 @@ are certain use cases that you may need to work around. For more information: - [`needs` requirements and limitations](../yaml/README.md#requirements-and-limitations). - Related epic [tracking planned improvements](https://gitlab.com/groups/gitlab-org/-/epics/1716). + +## DAG Visualization + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215517) in GitLab 13.1 as a [Beta feature](https://about.gitlab.com/handbook/product/#beta). +> - It's deployed behind a feature flag, disabled by default. +> - It's enabled on GitLab.com. +> - It's not recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [enable it](#enable-or-disable-dag-visualization-core-only) + +The DAG visualization makes it easier to visualize the relationships between dependent jobs in a DAG. This graph will display all the jobs in a pipeline that need or are needed by other jobs. Jobs with no relationships are not displayed in this view. + + + +Clicking a node will highlight all the job paths it depends on. + + + +### Enable or disable DAG Visualization **(CORE ONLY)** + +DAG Visualization is under development and requires more testing, but is being made available as a beta features so users can check its limitations and uses. + +It is deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can opt to enable it for your instance: + +```ruby +# Instance-wide +Feature.enable(:dag_pipeline_tab) +# or by project +Feature.enable(:dag_pipeline_tab, Project.find(<project id>)) +``` diff --git a/doc/ci/docker/README.md b/doc/ci/docker/README.md index f76471b50f2..4e7d9015e85 100644 --- a/doc/ci/docker/README.md +++ b/doc/ci/docker/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false type: index --- diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index f992af6c8a5..65b9c03186b 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -1,10 +1,13 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts, howto --- # Building Docker images with GitLab CI/CD -GitLab CI/CD allows you to use Docker Engine to build and test docker-based projects. +GitLab CI/CD allows you to use Docker Engine to build and test Docker-based projects. One of the new trends in Continuous Integration/Deployment is to: @@ -91,15 +94,15 @@ NOTE: **Note:** By adding `gitlab-runner` to the `docker` group you are effectively granting `gitlab-runner` full root permissions. For more information please read [On Docker security: `docker` group considered harmful](https://www.andreas-jung.com/contents/on-docker-security-docker-group-considered-harmful). -### Use docker-in-docker workflow with Docker executor +### Use Docker-in-Docker workflow with Docker executor -The second approach is to use the special docker-in-docker (dind) +The second approach is to use the special Docker-in-Docker (dind) [Docker image](https://hub.docker.com/_/docker/) with all tools installed (`docker`) and run the job script in context of that image in privileged mode. NOTE: **Note:** -`docker-compose` is not part of docker-in-docker (dind). To use `docker-compose` in your +`docker-compose` is not part of Docker-in-Docker (dind). To use `docker-compose` in your CI builds, follow the `docker-compose` [installation instructions](https://docs.docker.com/compose/install/). @@ -113,19 +116,19 @@ out the official Docker documentation on Docker-in-Docker works well, and is the recommended configuration, but it is not without its own challenges: -- When using docker-in-docker, each job is in a clean environment without the past +- When using Docker-in-Docker, each job is in a clean environment without the past history. Concurrent jobs work fine because every build gets its own instance of Docker engine so they won't conflict with each other. But this also means that jobs can be slower because there's no caching of layers. - By default, Docker 17.09 and higher uses `--storage-driver overlay2` which is the recommended storage driver. See [Using the overlayfs driver](#use-the-overlayfs-driver) for details. -- Since the `docker:19.03.8-dind` container and the Runner container don't share their +- Since the `docker:19.03.11-dind` container and the Runner container don't share their root filesystem, the job's working directory can be used as a mount point for child containers. For example, if you have files you want to share with a child container, you may create a subdirectory under `/builds/$CI_PROJECT_PATH` and use it as your mount point (for a more thorough explanation, check [issue - #41227](https://gitlab.com/gitlab-org/gitlab-foss/issues/41227)): + #41227](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227)): ```yaml variables: @@ -139,7 +142,7 @@ not without its own challenges: An example project using this approach can be found here: <https://gitlab.com/gitlab-examples/docker>. In the examples below, we are using Docker images tags to specify a -specific version, such as `docker:19.03.8`. If tags like `docker:stable` +specific version, such as `docker:19.03.11`. If tags like `docker:stable` are used, you have no control over what version is going to be used and this can lead to unpredictable behavior, especially when new versions are released. @@ -151,12 +154,12 @@ Requires GitLab Runner 11.11 or later, but is not supported if GitLab Runner is installed using the [Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). See the [related -issue](https://gitlab.com/gitlab-org/charts/gitlab-runner/issues/83) for +issue](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/83) for details. The Docker daemon supports connection over TLS and it's done by default -for Docker 19.03.8 or higher. This is the **suggested** way to use the -docker-in-docker service and +for Docker 19.03.11 or higher. This is the **suggested** way to use the +Docker-in-Docker service and [GitLab.com Shared Runners](../../user/gitlab_com/index.md#shared-runners) support this. @@ -171,19 +174,19 @@ support this. --registration-token REGISTRATION_TOKEN \ --executor docker \ --description "My Docker Runner" \ - --docker-image "docker:19.03.8" \ + --docker-image "docker:19.03.11" \ --docker-privileged \ --docker-volumes "/certs/client" ``` The above command will register a new Runner to use the special - `docker:19.03.8` image, which is provided by Docker. **Notice that it's + `docker:19.03.11` image, which is provided by Docker. **Notice that it's using the `privileged` mode to start the build and service - containers.** If you want to use [docker-in-docker](https://www.docker.com/blog/docker-can-now-run-within-docker/) mode, you always + containers.** If you want to use [Docker-in-Docker](https://www.docker.com/blog/docker-can-now-run-within-docker/) mode, you always have to use `privileged = true` in your Docker containers. This will also mount `/certs/client` for the service and build - container, which is needed for the docker client to use the + container, which is needed for the Docker client to use the certificates inside of that directory. For more information how Docker with TLS works check <https://hub.docker.com/_/docker/#tls>. @@ -196,7 +199,7 @@ support this. executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.8" + image = "docker:19.03.11" privileged = true disable_cache = false volumes = ["/certs/client", "/cache"] @@ -206,10 +209,10 @@ support this. ``` 1. You can now use `docker` in the build script (note the inclusion of the - `docker:19.03.8-dind` service): + `docker:19.03.11-dind` service): ```yaml - image: docker:19.03.8 + image: docker:19.03.11 variables: # When using dind service, we need to instruct docker, to talk with @@ -234,7 +237,7 @@ support this. DOCKER_TLS_CERTDIR: "/certs" services: - - docker:19.03.8-dind + - docker:19.03.11-dind before_script: - docker info @@ -261,7 +264,7 @@ Assuming that the Runner `config.toml` is similar to: executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.8" + image = "docker:19.03.11" privileged = true disable_cache = false volumes = ["/cache"] @@ -271,10 +274,10 @@ Assuming that the Runner `config.toml` is similar to: ``` You can now use `docker` in the build script (note the inclusion of the -`docker:19.03.8-dind` service): +`docker:19.03.11-dind` service): ```yaml -image: docker:19.03.8 +image: docker:19.03.11 variables: # When using dind service we need to instruct docker, to talk with the @@ -295,7 +298,7 @@ variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.8-dind + - docker:19.03.11-dind before_script: - docker info @@ -315,7 +318,7 @@ container so that Docker is available in the context of that image. NOTE: **Note:** If you bind the Docker socket [when using GitLab Runner 11.11 or newer](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1261), -you can no longer use `docker:19.03.8-dind` as a service because volume bindings +you can no longer use `docker:19.03.11-dind` as a service because volume bindings are done to the services as well, making these incompatible. In order to do that, follow the steps: @@ -330,12 +333,12 @@ In order to do that, follow the steps: --registration-token REGISTRATION_TOKEN \ --executor docker \ --description "My Docker Runner" \ - --docker-image "docker:19.03.8" \ + --docker-image "docker:19.03.11" \ --docker-volumes /var/run/docker.sock:/var/run/docker.sock ``` The above command will register a new Runner to use the special - `docker:19.03.8` image which is provided by Docker. **Notice that it's using + `docker:19.03.11` image which is provided by Docker. **Notice that it's using the Docker daemon of the Runner itself, and any containers spawned by Docker commands will be siblings of the Runner rather than children of the Runner.** This may have complications and limitations that are unsuitable for your workflow. @@ -349,7 +352,7 @@ In order to do that, follow the steps: executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.8" + image = "docker:19.03.11" privileged = false disable_cache = false volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"] @@ -358,11 +361,11 @@ In order to do that, follow the steps: ``` 1. You can now use `docker` in the build script (note that you don't need to - include the `docker:19.03.8-dind` service as when using the Docker in Docker + include the `docker:19.03.11-dind` service as when using the Docker in Docker executor): ```yaml - image: docker:19.03.8 + image: docker:19.03.11 before_script: - docker info @@ -377,7 +380,7 @@ In order to do that, follow the steps: While the above method avoids using Docker in privileged mode, you should be aware of the following implications: -- By sharing the docker daemon, you are effectively disabling all +- By sharing the Docker daemon, you are effectively disabling all the security mechanisms of containers and exposing your host to privilege escalation which can lead to container breakout. For example, if a project ran `docker rm -f $(docker ps -a -q)` it would remove the GitLab Runner @@ -392,9 +395,9 @@ aware of the following implications: docker run --rm -t -i -v $(pwd)/src:/home/app/src test-image:latest run_app_tests ``` -## Making docker-in-docker builds faster with Docker layer caching +## Making Docker-in-Docker builds faster with Docker layer caching -When using docker-in-docker, Docker will download all layers of your image every +When using Docker-in-Docker, Docker will download all layers of your image every time you create a build. Recent versions of Docker (Docker 1.13 and above) can use a pre-existing image as a cache during the `docker build` step, considerably speeding up the build process. @@ -416,10 +419,10 @@ any image that's used with the `--cache-from` argument must first be pulled Here's a `.gitlab-ci.yml` file showing how Docker caching can be used: ```yaml -image: docker:19.03.8 +image: docker:19.03.11 services: - - docker:19.03.8-dind + - docker:19.03.11-dind variables: # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled @@ -514,7 +517,7 @@ Once you've built a Docker image, you can push it up to the built-in ## Troubleshooting -### docker: Cannot connect to the Docker daemon at tcp://docker:2375. Is the docker daemon running? +### `docker: Cannot connect to the Docker daemon at tcp://docker:2375. Is the docker daemon running?` This is a common error when you are using [Docker in Docker](#use-docker-in-docker-workflow-with-docker-executor) diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 51139da2d16..2448bb536ab 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts, howto --- @@ -364,7 +367,7 @@ For example, the following two definitions are equal: | `alias` | no | 9.4 |Additional alias that can be used to access the service from the job's container. Read [Accessing the services](#accessing-the-services) for more information. | NOTE: **Note:** -Alias support for the Kubernetes executor was [introduced](https://gitlab.com/gitlab-org/gitlab-runner/issues/2229) in GitLab Runner 12.8, and is only available for Kubernetes version 1.7 or later. +Alias support for the Kubernetes executor was [introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2229) in GitLab Runner 12.8, and is only available for Kubernetes version 1.7 or later. ### Starting multiple services from the same image @@ -543,7 +546,7 @@ runtime. of credentials on runner's host. We recommend to upgrade your Runner to at least version **1.8** if you want to use private registries. - Not available for [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html), - follow <https://gitlab.com/gitlab-org/gitlab-runner/issues/2673> for + follow <https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2673> for details. ### Using statically-defined credentials @@ -591,7 +594,7 @@ There are two ways to determine the value of `DOCKER_AUTH_CONFIG`: ``` - **Second way -** In some setups, it's possible that Docker client - will use the available system keystore to store the result of `docker + will use the available system key store to store the result of `docker login`. In that case, it's impossible to read `~/.docker/config.json`, so you will need to prepare the required base64-encoded version of `${username}:${password}` and create the Docker configuration JSON manually. @@ -709,7 +712,7 @@ To configure credentials store, follow these steps: ``` - Or, if you are running self-managed Runners, add the above JSON to - `${GITLAB_RUNNER_HOME}/.docker/config.json`. GitLab Runner will read this config file + `${GITLAB_RUNNER_HOME}/.docker/config.json`. GitLab Runner will read this configuration file and will use the needed helper for this specific repository. NOTE: **Note:** `credsStore` is used to access ALL the registries. @@ -727,6 +730,9 @@ To configure access for `aws_account_id.dkr.ecr.region.amazonaws.com`, follow th 1. Make sure `docker-credential-ecr-login` is available in GitLab Runner's `$PATH`. +1. Have any of the following [AWS credentials setup](https://github.com/awslabs/amazon-ecr-credential-helper#aws-credentials). + Make sure that GitLab Runner can access the credentials. + 1. Make GitLab Runner use it. There are two ways to accomplish this. Either: - Create a [variable](../variables/README.md#gitlab-cicd-environment-variables) @@ -741,9 +747,21 @@ To configure access for `aws_account_id.dkr.ecr.region.amazonaws.com`, follow th } ``` + This configures Docker to use the credential helper for a specific registry. + + or + + ```json + { + "credsStore": "ecr-login" + } + ``` + + This configures Docker to use the credential helper for all Amazon ECR registries. + - Or, if you are running self-managed Runners, add the above JSON to `${GITLAB_RUNNER_HOME}/.docker/config.json`. - GitLab Runner will read this config file and will use the needed helper for this + GitLab Runner will read this configuration file and will use the needed helper for this specific repository. 1. You can now use any private image from `aws_account_id.dkr.ecr.region.amazonaws.com` defined in diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index 587f1f91f72..d53430400ec 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -1,21 +1,24 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- # Building images with kaniko and GitLab CI/CD -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/45512) in GitLab 11.2. Requires GitLab Runner 11.2 and above. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45512) in GitLab 11.2. Requires GitLab Runner 11.2 and above. [kaniko](https://github.com/GoogleContainerTools/kaniko) is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster. kaniko solves two problems with using the -[docker-in-docker +[Docker-in-Docker build](using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor) method: -- Docker-in-docker requires [privileged mode](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities) +- Docker-in-Docker requires [privileged mode](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities) in order to function, which is a significant security concern. -- Docker-in-docker generally incurs a performance penalty and can be quite slow. +- Docker-in-Docker generally incurs a performance penalty and can be quite slow. ## Requirements diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md index 12ed41e81ee..46cf76637a4 100644 --- a/doc/ci/enable_or_disable_ci.md +++ b/doc/ci/enable_or_disable_ci.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md new file mode 100644 index 00000000000..055baa78743 --- /dev/null +++ b/doc/ci/environments/deployment_safety.md @@ -0,0 +1,106 @@ +# Deployment safety + +Deployment jobs can be more sensitive than other jobs in a pipeline, +and might need to be treated with extra care. GitLab has several features +that help maintain deployment security and stability. + +You can: + +- [Restrict write-access to a critical environment](#restrict-write-access-to-a-critical-environment) +- [Restrict deployments for a particular period](#restrict-deployments-for-a-particular-period) + +If you are using a continuous deployment workflow and want to ensure that concurrent deployments to the same environment do not happen, you should enable the following options: + +- [Ensure only one deployment job runs at a time](#ensure-only-one-deployment-job-runs-at-a-time) +- [Skip outdated deployment jobs](#skip-outdated-deployment-jobs) + +## Restrict write access to a critical environment + +By default, environments can be modified by any team member that has [Developer permission or higher](../../user/permissions.md#project-members-permissions). +If you want to restrict write access to a critical environment (for example a `production` environment), +you can set up [protected environments](protected_environments.md). + +## Ensure only one deployment job runs at a time + +Pipeline jobs in GitLab CI/CD run in parallel, so it's possible that two deployment +jobs in two different pipelines attempt to deploy to the same environment at the same +time. This is not desired behavior as deployments should happen sequentially. + +You can ensure only one deployment job runs at a time with the [`resource_group` keyword](../yaml/README.md#resource_group) keyword in your `.gitlab-ci.yml`. + +For example: + +```yaml +deploy: + script: deploy-to-prod + resource_group: prod +``` + +Example of a problematic pipeline flow **before** using the resource group: + +1. `deploy` job in Pipeline-A starts running. +1. `deploy` job in Pipeline-B starts running. *This is a concurrent deployment that could cause an unexpected result.* +1. `deploy` job in Pipeline-A finished. +1. `deploy` job in Pipeline-B finished. + +The improved pipeline flow **after** using the resource group: + +1. `deploy` job in Pipeline-A starts running. +1. `deploy` job in Pipeline-B attempts to start, but waits for the first `deploy` job to finish. +1. `deploy` job in Pipeline-A finishes. +1. `deploy` job in Pipeline-B starts running. + +For more information, see [`resource_group` keyword in `.gitlab-ci.yml`](../yaml/README.md#resource_group). + +## Skip outdated deployment jobs + +The execution order of pipeline jobs can vary from run to run, which could cause +undesired behavior. For example, a deployment job in a newer pipeline could +finish before a deployment job in an older pipeline. +This creates a race condition where the older deployment finished later, +overwriting the "newer" deployment. + +You can ensure that older deployment jobs are cancelled automatically when a newer deployment +runs by enabling the [Skip outdated deployment jobs](../pipelines/settings.md#skip-outdated-deployment-jobs) feature. + +Example of a problematic pipeline flow **before** enabling Skip outdated deployment jobs: + +1. Pipeline-A is created on the master branch. +1. Later, Pipeline-B is created on the master branch (with a newer commit SHA). +1. The `deploy` job in Pipeline-B finishes first, and deploys the newer code. +1. The `deploy` job in Pipeline-A finished later, and deploys the older code, **overwriting** the newer (latest) deployment. + +The improved pipeline flow **after** enabling Skip outdated deployment jobs: + +1. Pipeline-A is created on the `master` branch. +1. Later, Pipeline-B is created on the `master` branch (with a newer SHA). +1. The `deploy` job in Pipeline-B finishes first, and deploys the newer code. +1. The `deploy` job in Pipeline-A is automatically cancelled, so that it doesn't overwrite the deployment from the newer pipeline. + +## Restrict deployments for a particular period + +If you want to prevent deployments for a particular period, for example during a planned +vacation period when most employees are out, you can set up a [Deploy Freeze](../../user/project/releases/index.md#set-a-deploy-freeze). +During a deploy freeze period, no deployment can be executed. This is helpful to +ensure that deployments do not happen unexpectedly. + +## Troubleshooting + +### Pipelines jobs fail with `The deployment job is older than the previously succeeded deployment job...` + +This is caused by the [Skip outdated deployment jobs](../pipelines/settings.md#skip-outdated-deployment-jobs) feature. +If you have multiple jobs for the same environment (including non-deployment jobs), you might encounter this problem, for example: + +```yaml +build:service-a: + environment: + name: production + +build:service-b: + environment: + name: production +``` + +The [Skip outdated deployment jobs](../pipelines/settings.md#skip-outdated-deployment-jobs) might not work well with this configuration, and will need to be disabled. + +There is a [plan to introduce a new annotation for environments](https://gitlab.com/gitlab-org/gitlab/-/issues/208655) to address this issue. diff --git a/doc/ci/environments/environments_dashboard.md b/doc/ci/environments/environments_dashboard.md index 4a72c0d6d62..e920e0d2400 100644 --- a/doc/ci/environments/environments_dashboard.md +++ b/doc/ci/environments/environments_dashboard.md @@ -7,7 +7,7 @@ type: reference # Environments Dashboard **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3713) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3713) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. The Environments Dashboard provides a cross-project environment-based view that lets you see the big picture diff --git a/doc/ci/environments/incremental_rollouts.md b/doc/ci/environments/incremental_rollouts.md index 016a6ac7cad..5da5c8e0a87 100644 --- a/doc/ci/environments/incremental_rollouts.md +++ b/doc/ci/environments/incremental_rollouts.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: concepts, howto --- @@ -34,7 +37,7 @@ use as examples to build your own: ## Manual Rollouts -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5415) in GitLab 10.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5415) in GitLab 10.8. It is possible to configure GitLab to do incremental rollouts manually through `.gitlab-ci.yml`. Manual configuration allows more control over the this feature. The steps in an incremental rollout depend on the @@ -74,7 +77,7 @@ available, demonstrating manually triggered incremental rollouts. ## Timed Rollouts -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7545) in GitLab 11.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7545) in GitLab 11.4. Timed rollouts behave in the same way as manual rollouts, except that each job is defined with a delay in minutes before it will deploy. Clicking on the job will reveal the countdown. @@ -111,3 +114,26 @@ timed rollout 30%: A [deployable application](https://gitlab.com/gl-release/timed-rollout-example) is available, [demonstrating configuration of timed rollouts](https://gitlab.com/gl-release/timed-rollout-example/blob/master/.gitlab-ci.yml#L86-95). + +## Blue-Green Deployment + +Also sometimes known as A/B deployment or red-black deployment, this technique is used to reduce +downtime and risk during a deployment. When combined with incremental rollouts, you can +minimize the impact of a deployment causing an issue. + +With this technique there are two deployments ("blue" and "green", but any naming can be used). +Only one of these deployments is live at any given time, except during an incremental rollout. + +For example, your blue deployment can be currently active on production, while the +green deployment is "live" for testing, but not deployed to production. If issues +are found, the green deployment can be updated without affecting the production +deployment (currently blue). If testing finds no issues, you switch production to the green +deployment, and blue is now available to test the next release. + +This process reduces downtime as there is no need to take down the production deployment +to switch to a different deployment. Both deployments are running in parallel, and +can be switched to at any time. + +An [example deployable application](https://gitlab.com/gl-release/blue-green-example) +is available, with a [`gitlab-ci.yml` CI/CD configuration file](https://gitlab.com/gl-release/blue-green-example/blob/master/.gitlab-ci.yml) +that demonstrates blue-green deployments. diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index b6ec30b5571..84480b836f8 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -162,7 +162,7 @@ Starting with GitLab 9.3, the environment URL is exposed to the Runner via #### Set dynamic environment URLs after a job finishes -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17066) in GitLab 12.9. In a job script, you can specify a static [environment URL](#using-the-environment-url). However, there may be times when you want a dynamic URL. For example, @@ -384,7 +384,7 @@ feature. ### Configuring Kubernetes deployments -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27630) in GitLab 12.6. If you are deploying to a [Kubernetes cluster](../../user/project/clusters/index.md) associated with your project, you can configure these deployments from your @@ -421,7 +421,18 @@ NOTE: **Note:** Kubernetes configuration is not supported for Kubernetes clusters that are [managed by GitLab](../../user/project/clusters/index.md#gitlab-managed-clusters). To follow progress on support for GitLab-managed clusters, see the -[relevant issue](https://gitlab.com/gitlab-org/gitlab/issues/38054). +[relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). + +### Deployment safety + +Deployment jobs can be more sensitive than other jobs in a pipeline, +and might need to be treated with an extra care. There are multiple features +in GitLab that helps maintain deployment security and stability. + +- [Restrict write-access to a critical environment](deployment_safety.md#restrict-write-access-to-a-critical-environment) +- [Limit the job-concurrency for deployment jobs](deployment_safety.md#ensure-only-one-deployment-job-runs-at-a-time) +- [Skip outdated deployment jobs](deployment_safety.md#skip-outdated-deployment-jobs) +- [Restrict deployments for a particular period](deployment_safety.md#restrict-deployments-for-a-particular-period) ### Complete example @@ -699,11 +710,17 @@ stop action when the associated branch is deleted. The `stop_review` job must be in the same `stage` as the `deploy_review` job in order for the environment to automatically stop. +Additionally, both jobs should have matching [`rules`](../yaml/README.md#onlyexcept-basic) +or [`only/except`](../yaml/README.md#onlyexcept-basic) configuration. In the example +above, if the configuration is not identical, the `stop_review` job might not be +included in all pipelines that include the `deploy_review` job, and it will not be +possible to trigger the `action: stop` to stop the environment automatically. + You can read more in the [`.gitlab-ci.yml` reference](../yaml/README.md#environmenton_stop). #### Environments auto-stop -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/20956) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20956) in GitLab 12.8. You can set a expiry time to environments and stop them automatically after a certain period. @@ -849,10 +866,6 @@ versions of the app, all without leaving GitLab.  -#### Linking to external dashboard - -Add a [button to the Monitoring dashboard](../../user/project/operations/linking_to_an_external_dashboard.md) linking directly to your existing external dashboards. - #### Embedding metrics in GitLab Flavored Markdown Metric charts can be embedded within GitLab Flavored Markdown. See [Embedding Metrics within GitLab Flavored Markdown](../../user/project/integrations/prometheus.md#embedding-metric-charts-within-gitlab-flavored-markdown) for more details. @@ -912,7 +925,7 @@ fetch = +refs/environments/*:refs/remotes/origin/environments/* ### Scoping environments with specs > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2112) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.4. -> - [Scoping for environment variables was moved to Core](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30779) to Core in GitLab 12.2. +> - [Scoping for environment variables was moved to Core](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30779) in GitLab 12.2. You can limit the environment scope of a variable by defining which environments it can be available for. diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index 43ac42ea0c7..3a44fcfb182 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -55,6 +55,8 @@ Maintainers can: **Allowed to Deploy** dropdown menu. - Unprotect a protected environment by clicking the **Unprotect** button for that environment. +For more information, see [Deployment safety](deployment_safety.md). + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/ci/examples/README.md b/doc/ci/examples/README.md index 1b3c4c887f4..c54e4b96332 100644 --- a/doc/ci/examples/README.md +++ b/doc/ci/examples/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false type: index --- @@ -10,9 +13,8 @@ implement [GitLab CI/CD](../README.md) for your specific use case. Examples are available in several forms. As a collection of: -- `.gitlab-ci.yml` [template files](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/lib/gitlab/ci/templates) maintained in GitLab. When you create a new file via the UI, - GitLab will give you the option to choose one of these templates. This will allow you to start using CI/CD with your project quickly. - If your favorite programming language or framework are missing, we would love your help by sending a merge request with a new `.gitlab-ci.yml` to this project. +- `.gitlab-ci.yml` [template files](#cicd-templates) maintained in GitLab, for many + common frameworks and programming languages. - Repositories with [example projects](https://gitlab.com/gitlab-examples) for various languages. You can fork and adjust them to your own needs. Projects include demonstrations of [multi-project pipelines](https://gitlab.com/gitlab-examples/multi-project-pipelines) and using [Review Apps with a static site served by NGINX](https://gitlab.com/gitlab-examples/review-apps-nginx/). - Examples and [other resources](#other-resources) listed below. @@ -47,9 +49,55 @@ language users and GitLab by sending a merge request with a guide for that langu You may want to apply for the [GitLab Community Writers Program](https://about.gitlab.com/community/writers/) to get paid for writing complete articles for GitLab. -## Adding templates to your GitLab installation **(PREMIUM ONLY)** - -If you want to have customized examples and templates for your own self-managed GitLab instance available to your team, your GitLab administrator can [designate an instance template repository](../../user/admin_area/settings/instance_template_repository.md) that contains examples and templates specific to your enterprise. +## CI/CD templates + +Get started with GitLab CI/CD and your favorite programming language or framework by using a +`.gitlab-ci.yml` [template](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates). + +When you create a `gitlab-ci.yml` file in the UI, you can +choose one of these templates: + +- [Android (`Android.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android.gitlab-ci.yml) +- [Android with fastlane (`Android-Fastlane.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml) +- [Bash (`Bash.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml) +- [C++ (`C++.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/C++.gitlab-ci.yml) +- [Chef (`Chef.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml) +- [Clojure (`Clojure.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml) +- [Crystal (`Crystal.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml) +- [Django (`Django.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Django.gitlab-ci.yml) +- [Docker (`Docker.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml) +- [dotNET (`dotNET.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml) +- [dotNET Core (`dotNET-Core.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml) +- [Elixir (`Elixir.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml) +- [goLang (`Go.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Go.gitlab-ci.yml) +- [Gradle (`Gradle.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml) +- [Grails (`Grails.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml) +- [iOS with fastlane (`iOS-Fastlane.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml) +- [Julia (`Julia.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml) +- [Laravel (`Laravel.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml) +- [LaTeX (`LaTeX.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml) +- [Maven (`Maven.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Maven.gitlab-ci.yml) +- [Mono (`Mono.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml) +- [Node.js (`Nodejs.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml) +- [OpenShift (`OpenShift.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml) +- [Packer (`Packer.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Packer.gitlab-ci.yml) +- [PHP (`PHP.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/PHP.gitlab-ci.yml) +- [Python (`Python.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml) +- [Ruby (`Ruby.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml) +- [Rust (`Rust.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Rust.gitlab-ci.yml) +- [Scala (`Scala.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Scala.gitlab-ci.yml) +- [Swift (`Swift.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Swift.gitlab-ci.yml) +- [Terraform (`Terraform.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) + +If a programming language or framework template is not in this list, you can contribute +one. To create a template, submit a merge request +to <https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates>. + +### Adding templates to your GitLab installation **(PREMIUM ONLY)** + +You can add custom examples and templates to your self-managed GitLab instance. +Your GitLab administrator can [designate an instance template repository](../../user/admin_area/settings/instance_template_repository.md) +that contains examples and templates specific to your organization. ## Other resources diff --git a/doc/ci/examples/artifactory_and_gitlab/index.md b/doc/ci/examples/artifactory_and_gitlab/index.md index 85dcc199e2b..c1b3ddec1b9 100644 --- a/doc/ci/examples/artifactory_and_gitlab/index.md +++ b/doc/ci/examples/artifactory_and_gitlab/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/articles/artifactory_and_gitlab/index.html' author: Fabio Busatto author_gitlab: bikebilly @@ -106,7 +109,7 @@ parameter in `.gitlab-ci.yml` to use the custom location instead of the default Now it's time we set up [GitLab CI/CD](https://about.gitlab.com/stages-devops-lifecycle/continuous-integration/) to automatically build, test and deploy the dependency! -GitLab CI/CD uses a file in the root of the repo, named `.gitlab-ci.yml`, to read the definitions for jobs +GitLab CI/CD uses a file in the root of the repository, named `.gitlab-ci.yml`, to read the definitions for jobs that will be executed by the configured GitLab Runners. You can read more about this file in the [GitLab Documentation](../../yaml/README.md). First of all, remember to set up variables for your deployment. Navigate to your project's **Settings > CI/CD > Environment variables** page @@ -116,7 +119,7 @@ and add the following ones (replace them with your current values, of course): - **MAVEN_REPO_USER**: `gitlab` (your Artifactory username) - **MAVEN_REPO_PASS**: `AKCp2WXr3G61Xjz1PLmYa3arm3yfBozPxSta4taP3SeNu2HPXYa7FhNYosnndFNNgoEds8BCS` (your Artifactory Encrypted Password) -Now it's time to define jobs in `.gitlab-ci.yml` and push it to the repo: +Now it's time to define jobs in `.gitlab-ci.yml` and push it to the repository: ```yaml image: maven:latest @@ -151,7 +154,7 @@ deploy: GitLab Runner will use the latest [Maven Docker image](https://hub.docker.com/_/maven/), which already contains all the tools and the dependencies you need to manage the project, in order to run the jobs. -Environment variables are set to instruct Maven to use the `homedir` of the repo instead of the user's home when searching for configuration and dependencies. +Environment variables are set to instruct Maven to use the `homedir` of the repository instead of the user's home when searching for configuration and dependencies. Caching the `.m2/repository folder` (where all the Maven files are stored), and the `target` folder (where our application will be created), is useful for speeding up the process by running all Maven phases in a sequential order, therefore, executing `mvn test` will automatically run `mvn compile` if necessary. @@ -161,7 +164,7 @@ Both `build` and `test` jobs leverage the `mvn` command to compile the applicati Deploy to Artifactory is done as defined by the variables we have just set up. The deployment occurs only if we're pushing or merging to `master` branch, so that the development versions are tested but not published. -Done! Now you have all the changes in the GitLab repo, and a pipeline has already been started for this commit. In the **Pipelines** tab you can see what's happening. +Done! Now you have all the changes in the GitLab repository, and a pipeline has already been started for this commit. In the **Pipelines** tab you can see what's happening. If the deployment has been successful, the deploy job log will output: ```plaintext @@ -174,7 +177,7 @@ If the deployment has been successful, the deploy job log will output: >**Note**: the `mvn` command downloads a lot of files from the internet, so you'll see a lot of extra activity in the log the first time you run it. -Yay! You did it! Checking in Artifactory will confirm that you have a new artifact available in the `libs-release-local` repo. +Yay! You did it! Checking in Artifactory will confirm that you have a new artifact available in the `libs-release-local` repository. ## Create the main Maven application @@ -225,7 +228,7 @@ Here is how you can get the content of the file directly from Artifactory: 1. Click on **Generate Maven Settings** 1. Click on **Generate Settings** 1. Copy to clipboard the configuration file -1. Save the file as `.m2/settings.xml` in your repo +1. Save the file as `.m2/settings.xml` in your repository Now you are ready to use the Artifactory repository to resolve dependencies and use `simple-maven-dep` in your main application! @@ -236,7 +239,7 @@ You need a last step to have everything in place: configure the `.gitlab-ci.yml` You want to leverage [GitLab CI/CD](https://about.gitlab.com/stages-devops-lifecycle/continuous-integration/) to automatically build, test and run your awesome application, and see if you can get the greeting as expected! -All you need to do is to add the following `.gitlab-ci.yml` to the repo: +All you need to do is to add the following `.gitlab-ci.yml` to the repository: ```yaml image: maven:latest diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md index 089babc8945..3b8be54ae59 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Release Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- diff --git a/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md b/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md index 49f4a14c5ac..87cee1820bc 100644 --- a/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md +++ b/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers author: Dylan Griffith author_gitlab: DylanGriffith level: intermediate @@ -100,7 +103,7 @@ production: - master ``` -We've used the `java:8` [docker +We've used the `java:8` [Docker image](../../docker/using_docker_images.md) to build our application as it provides the up-to-date Java 8 JDK on [Docker Hub](https://hub.docker.com/). We've also added the [`only` diff --git a/doc/ci/examples/deployment/README.md b/doc/ci/examples/deployment/README.md index 2ed1a99e0d9..ec02fb6dd43 100644 --- a/doc/ci/examples/deployment/README.md +++ b/doc/ci/examples/deployment/README.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- @@ -53,7 +56,7 @@ To use different provider take a look at long list of [Supported Providers](http ## Using Dpl with Docker In most cases, you will have configured [GitLab Runner](https://docs.gitlab.com/runner/) to use your server's shell commands. -This means that all commands are run in the context of local user (e.g. gitlab_runner or gitlab_ci_multi_runner). +This means that all commands are run in the context of local user (e.g. `gitlab_runner` or `gitlab_ci_multi_runner`). It also means that most probably in your Docker container you don't have the Ruby runtime installed. You will have to install it: diff --git a/doc/ci/examples/deployment/composer-npm-deploy.md b/doc/ci/examples/deployment/composer-npm-deploy.md index c5f49fd6e59..cea6f26181f 100644 --- a/doc/ci/examples/deployment/composer-npm-deploy.md +++ b/doc/ci/examples/deployment/composer-npm-deploy.md @@ -1,4 +1,7 @@ --- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- @@ -44,7 +47,7 @@ All these operations will put all files into a `build` folder, which is ready to ## How to transfer files to a live server -You have multiple options: rsync, scp, sftp, and so on. For now, we will use scp. +You have multiple options: rsync, SCP, SFTP, and so on. For now, we will use SCP. To make this work, you need to add a GitLab CI/CD Variable (accessible on `gitlab.example/your-project-name/variables`). That variable will be called `STAGING_PRIVATE_KEY` and it's the **private** SSH key of your server. @@ -74,7 +77,7 @@ And this is basically all you need in the `before_script` section. ## How to deploy -As we stated above, we need to deploy the `build` folder from the docker image to our server. To do so, we create a new job: +As we stated above, we need to deploy the `build` folder from the Docker image to our server. To do so, we create a new job: ```yaml stage_deploy: @@ -94,7 +97,7 @@ stage_deploy: Here's the breakdown: 1. `only:dev` means that this build will run only when something is pushed to the `dev` branch. You can remove this block completely and have everything be ran on every push (but probably this is something you don't want) -1. `ssh-add ...` we will add that private key you added on the web UI to the docker container +1. `ssh-add ...` we will add that private key you added on the web UI to the Docker container 1. We will connect via `ssh` and create a new `_tmp` folder 1. We will connect via `scp` and upload the `build` folder (which was generated by a `npm` script) to our previously created `_tmp` folder 1. We will connect again via `ssh` and move the `live` folder to an `_old` folder, then move `_tmp` to `live`. @@ -120,7 +123,7 @@ Therefore, for a production environment we use additional steps to ensure that a Since this was a WordPress project, I gave real life code snippets. Some further ideas you can pursue: - Having a slightly different script for `master` branch will allow you to deploy to a production server from that branch and to a stage server from any other branches. -- Instead of pushing it live, you can push it to WordPress official repo (with creating a SVN commit, etc.). +- Instead of pushing it live, you can push it to WordPress official repository (with creating a SVN commit, etc.). - You could generate i18n text domains on the fly. --- diff --git a/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md b/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md index 610220a6bff..51d9f169939 100644 --- a/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md +++ b/doc/ci/examples/devops_and_game_dev_with_gitlab_ci_cd/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers author: Ryan Hall author_gitlab: blitzgren level: intermediate @@ -353,7 +356,7 @@ test: ### Run your CI/CD pipeline -That's it! Add all your new files, commit, and push. For a reference of what our repo should +That's it! Add all your new files, commit, and push. For a reference of what our repository should look like at this point, please refer to the [final commit related to this article on my sample repository](https://gitlab.com/blitzgren/gitlab-game-demo/commit/8b36ef0ecebcf569aeb251be4ee13743337fcfe2). By applying both build and test stages, GitLab will run them sequentially at every push to our repository. If all goes well you'll end up with a green check mark on each job for the pipeline: @@ -419,15 +422,15 @@ fully understand [IAM Best Practices in AWS](https://docs.aws.amazon.com/IAM/lat 1. Log into your AWS account and go to the [Security Credentials page](https://console.aws.amazon.com/iam/home#/security_credential) 1. Click the **Access Keys** section and **Create New Access Key**. Create the key and keep the ID and secret around, you'll need them later -  +  1. Go to your GitLab project, click **Settings > CI/CD** on the left sidebar 1. Expand the **Variables** section  -1. Add a key named `AWS_KEY_ID` and copy the key ID from Step 2 into the **Value** textbox -1. Add a key named `AWS_KEY_SECRET` and copy the key secret from Step 2 into the **Value** textbox +1. Add a key named `AWS_KEY_ID` and copy the key ID from Step 2 into the **Value** field +1. Add a key named `AWS_KEY_SECRET` and copy the key secret from Step 2 into the **Value** field ### Deploy your game with GitLab CI/CD @@ -526,7 +529,7 @@ a lot of breathing room in quickly getting changes to players. Here are some ideas to further investigate that can speed up or improve your pipeline: - [Yarn](https://yarnpkg.com) instead of npm -- Set up a custom [Docker](../../../ci/docker/using_docker_images.md#define-image-and-services-from-gitlab-ciyml) image that can preload dependencies and tools (like AWS CLI) +- Set up a custom [Docker](../../../ci/docker/using_docker_images.md#define-image-and-services-from-gitlab-ciyml) image that can pre-load dependencies and tools (like AWS CLI) - Forward a [custom domain](https://docs.aws.amazon.com/AmazonS3/latest/dev/website-hosting-custom-domain-walkthrough.html) to your game's S3 static website - Combine jobs if you find it unnecessary for a small project - Avoid the queues and set up your own [custom GitLab CI/CD runner](https://about.gitlab.com/blog/2016/03/01/gitlab-runner-with-docker/) diff --git a/doc/ci/examples/end_to_end_testing_webdriverio/index.md b/doc/ci/examples/end_to_end_testing_webdriverio/index.md index 8491881570c..05b3cc257e3 100644 --- a/doc/ci/examples/end_to_end_testing_webdriverio/index.md +++ b/doc/ci/examples/end_to_end_testing_webdriverio/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers author: Vincent Tunru author_gitlab: Vinnl level: advanced diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index 0331fa70f06..5b442436baf 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/articles/laravel_with_gitlab_and_envoy/index.html' author: Mehran Rasulian author_gitlab: mehranrasulian @@ -29,7 +32,7 @@ It uses a clean, minimal [Blade syntax](https://laravel.com/docs/master/blade) t ## Initialize our Laravel app on GitLab -We assume [you have installed a new laravel project](https://laravel.com/docs/master/installation#installation), so let's start with a unit test, and initialize Git for the project. +We assume [you have installed a new Laravel project](https://laravel.com/docs/master/installation#installation), so let's start with a unit test, and initialize Git for the project. ### Unit Test @@ -664,7 +667,7 @@ If something doesn't work as expected, you can roll back to the latest working v By clicking on the external link icon specified on the right side, GitLab opens the production website. Our deployment successfully was done and we can see the application is live. - + In the case that you're interested to know how is the application directory structure on the production server after deployment, here are three directories named `current`, `releases` and `storage`. As you know, the `current` directory is a symbolic link that points to the latest release. diff --git a/doc/ci/examples/php.md b/doc/ci/examples/php.md index 6a6c0e8fde1..e7768868c15 100644 --- a/doc/ci/examples/php.md +++ b/doc/ci/examples/php.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- @@ -13,7 +16,7 @@ using the Shell executor. While it is possible to test PHP apps on any system, this would require manual configuration from the developer. To overcome this we will be using the -official [PHP docker image](https://hub.docker.com/_/php) that can be found in Docker Hub. +official [PHP Docker image](https://hub.docker.com/_/php) that can be found in Docker Hub. This will allow us to test PHP projects against different versions of PHP. However, not everything is plug 'n' play, you still need to configure some @@ -62,7 +65,7 @@ docker-php-ext-install pdo_mysql ``` You might wonder what `docker-php-ext-install` is. In short, it is a script -provided by the official php docker image that you can use to easily install +provided by the official PHP Docker image that you can use to easily install extensions. For more information read the documentation at <https://hub.docker.com/_/php>. @@ -111,7 +114,7 @@ test:app: ### Test against different PHP versions in Docker builds Testing against multiple versions of PHP is super easy. Just add another job -with a different docker image version and the runner will do the rest: +with a different Docker image version and the runner will do the rest: ```yaml before_script: @@ -171,7 +174,7 @@ Finally, push to GitLab and let the tests begin! ### Test against different PHP versions in Shell builds The [phpenv](https://github.com/phpenv/phpenv) project allows you to easily manage different versions of PHP -each with its own config. This is especially useful when testing PHP projects +each with its own configuration. This is especially useful when testing PHP projects with the Shell executor. You will have to install it on your build machine under the `gitlab-runner` diff --git a/doc/ci/examples/test-and-deploy-python-application-to-heroku.md b/doc/ci/examples/test-and-deploy-python-application-to-heroku.md index 2c626cb458a..30a64e65607 100644 --- a/doc/ci/examples/test-and-deploy-python-application-to-heroku.md +++ b/doc/ci/examples/test-and-deploy-python-application-to-heroku.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- diff --git a/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md b/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md index 5df407f19fe..bf589c5991d 100644 --- a/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md +++ b/doc/ci/examples/test-and-deploy-ruby-application-to-heroku.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- diff --git a/doc/ci/examples/test-clojure-application.md b/doc/ci/examples/test-clojure-application.md index 6ea38f22bca..31dacd34730 100644 --- a/doc/ci/examples/test-clojure-application.md +++ b/doc/ci/examples/test-clojure-application.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- diff --git a/doc/ci/examples/test-scala-application.md b/doc/ci/examples/test-scala-application.md index 1831417e48e..fec376915c9 100644 --- a/doc/ci/examples/test-scala-application.md +++ b/doc/ci/examples/test-scala-application.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- diff --git a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md index cd1ad923249..c901f6c9c66 100644 --- a/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md +++ b/doc/ci/examples/test_phoenix_app_with_gitlab_ci_cd/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers author: Alexandre S Hostert author_gitlab: Hostert level: beginner @@ -43,9 +46,9 @@ Phoenix can run in any OS where Erlang is supported: - Debian - Windows - Fedora -- Raspbian +- Raspberry Pi OS -Check the [Phoenix learning guide](https://hexdocs.pm/phoenix/learning.html) for more information. +Check the [Phoenix learning guide](https://hexdocs.pm/phoenix/overview.html) for more information. ### What is Elixir? @@ -151,7 +154,7 @@ point `localhost` to `127.0.0.1`. Great, now we have a local Phoenix Server running our app. -Locally, our application is running in an `iex` session. [iex](https://elixir-lang.org/getting-started/introduction.html#interactive-mode) stands for Interactive Elixir. +Locally, our application is running in an [`iex`](https://elixir-lang.org/getting-started/introduction.html#interactive-mode) session, which stands for Interactive Elixir. In this interactive mode, we can type any Elixir expression and get its result. To exit `iex`, we need to press `Ctrl+C` twice. So, when we need to stop the Phoenix server, we have to hit `Ctrl+C` twice. @@ -161,7 +164,7 @@ twice. With GitLab, we can manage our development workflow, improve our productivity, track issues, perform code review, and much more from a single platform. With GitLab CI/CD, we can be much more productive, because every time we, or our co-workers push any code, GitLab CI/CD will build and -test the changes, telling us in realtime if anything goes wrong. +test the changes, telling us in real time if anything goes wrong. Certainly, when our application starts to grow, we'll need more developers working on the same project and this process of building and testing can easily become a mess without proper management. @@ -262,7 +265,7 @@ project. our application? This virtual machine must have all dependencies to run our application. This is where a Docker image is needed. The correct image will provide the entire system for us. - As we are focusing on testing (not deploying), you can use the [elixir:latest](https://hub.docker.com/_/elixir) docker image, which already has the + As we are focusing on testing (not deploying), you can use the [elixir:latest](https://hub.docker.com/_/elixir) Docker image, which already has the dependencies for running Phoenix tests installed, such as Elixir and Erlang: ```yaml diff --git a/doc/ci/git_submodules.md b/doc/ci/git_submodules.md index 4797251129b..b12ac59625f 100644 --- a/doc/ci/git_submodules.md +++ b/doc/ci/git_submodules.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/ci/interactive_web_terminal/index.md b/doc/ci/interactive_web_terminal/index.md index 661e32b1571..be3741332e0 100644 --- a/doc/ci/interactive_web_terminal/index.md +++ b/doc/ci/interactive_web_terminal/index.md @@ -1,10 +1,13 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Interactive Web Terminals -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/50144) in GitLab 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/50144) in GitLab 11.3. Interactive web terminals give the user access to a terminal in GitLab for running one-off commands for their CI pipeline. Since this is giving the user @@ -31,7 +34,7 @@ Two things need to be configured for the interactive web terminal to work: NOTE: **Note:** Interactive web terminals are not yet supported by [`gitlab-runner` Helm chart](https://docs.gitlab.com/charts/charts/gitlab/gitlab-runner/index.html), -but support [is planned](https://gitlab.com/gitlab-org/charts/gitlab-runner/issues/79). +but support [is planned](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/79). ## Debugging a running job @@ -41,7 +44,7 @@ NOTE: **Note:** Not all executors are NOTE: **Note:** The `docker` executor does not keep running after the build script is finished. At that point, the terminal will automatically disconnect and will not wait for the user to finish. Please follow [this -issue](https://gitlab.com/gitlab-org/gitlab-runner/issues/3605) for updates on +issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3605) for updates on improving this behavior. Sometimes, when a job is running, things don't go as you would expect, and it diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index b7f1837d83e..7ba24f9c861 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers description: "An overview of Continuous Integration, Continuous Delivery, and Continuous Deployment, as well as an introduction to GitLab CI/CD." type: concepts --- diff --git a/doc/ci/jenkins/index.md b/doc/ci/jenkins/index.md index c4346005138..37e0522e74d 100644 --- a/doc/ci/jenkins/index.md +++ b/doc/ci/jenkins/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false type: index, howto --- @@ -72,13 +75,13 @@ There are some high level differences between the products worth mentioning: with the [`rules` syntax](../yaml/README.md#rules). - GitLab [pipeline scheduling concepts](../pipelines/schedules.md) are also different than with Jenkins. - You can reuse pipeline configurations using the [`include` keyword](../yaml/README.md#include) - and [templates](#templates). Your templates can be kept in a central repo (with different + and [templates](#templates). Your templates can be kept in a central repository (with different permissions), and then any project can use them. This central project could also contain scripts or other reusable code. - You can also use the [`extends` keyword](../yaml/README.md#extends) to reuse configuration within a single pipeline configuration. - All jobs within a single stage always run in parallel, and all stages run in sequence. We are planning - to allow certain jobs to break this sequencing as needed with our [directed acyclic graph](https://gitlab.com/gitlab-org/gitlab-foss/issues/47063) + to allow certain jobs to break this sequencing as needed with our [directed acyclic graph](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) feature. - The [`parallel`](../yaml/README.md#parallel) keyword can automatically parallelize tasks, like tests that support parallelization. @@ -90,7 +93,7 @@ There are some high level differences between the products worth mentioning: - One important difference is that jobs run independently of each other and have a fresh environment in each job. Passing artifacts between jobs is controlled using the [`artifacts`](../yaml/README.md#artifacts) and [`dependencies`](../yaml/README.md#dependencies) - keywords. When finished, the planned [Workspaces](https://gitlab.com/gitlab-org/gitlab/issues/29265) + keywords. When finished, the planned [Workspaces](https://gitlab.com/gitlab-org/gitlab/-/issues/29265) feature will allow you to more easily persist a common workspace between serial jobs. - The `.gitlab-ci.yml` file is checked in to the root of your repository, much like a Jenkinsfile, but is in the YAML format (see [complete reference](../yaml/README.md)) instead of a Groovy DSL. It's most @@ -115,13 +118,13 @@ agents you were using. There are some important differences in the way Runners work in comparison to agents: -- Runners can be set up as [shared across an instance, be added at the group level, or set up at the project level](../runners/README.md#shared-specific-and-group-runners). +- Runners can be set up as [shared across an instance, be added at the group level, or set up at the project level](../runners/README.md#types-of-runners). They will self-select jobs from the scopes you've defined automatically. -- You can also [use tags](../runners/README.md#using-tags) for finer control, and +- You can also [use tags](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) for finer control, and associate runners with specific jobs. For example, you can use a tag for jobs that require dedicated, more powerful, or specific hardware. - GitLab has [autoscaling for Runners](https://docs.gitlab.com/runner/configuration/autoscale.html) - which will let configure them to be provisioned as needed, and scaled down when not. + which will let you configure them to be provisioned as needed, and scaled down when not. This is similar to ephemeral agents in Jenkins. If you are using `gitlab.com`, you can take advantage of our [shared Runner fleet](../../user/gitlab_com/index.md#shared-runners) @@ -136,7 +139,7 @@ GitLab works a bit differently, we use the more highly structured [YAML](https:/ places scripting elements inside of `script:` blocks separate from the pipeline specification itself. This is a strength of GitLab, in that it helps keep the learning curve much simpler to get up and running -and avoids some of the problem of unconstrained complexity which can make your Jenkinsfiles hard to understand +and avoids some of the problem of unconstrained complexity which can make your Jenkinsfile hard to understand and manage. That said, we do of course still value DRY (don't repeat yourself) principles and want to ensure that @@ -202,12 +205,12 @@ be used by all projects in the group. An instance administrator can set a group the source for [instance project templates](../../user/group/custom_project_templates.md), which can be used by projects in that instance. -## Converting Declarative Jenkinsfiles +## Converting a declarative Jenkinsfile -Declarative Jenkinsfiles contain "Sections" and "Directives" which are used to control the behavior of your +A declarative Jenkinsfile contains "Sections" and "Directives" which are used to control the behavior of your pipelines. There are equivalents for all of these in GitLab, which we've documented below. -This section is based on the [Jenkinsfile syntax documentation](https://jenkins.io/doc/book/pipeline/syntax/) +This section is based on the [Jenkinsfile syntax documentation](https://www.jenkins.io/doc/book/pipeline/syntax/) and is meant to be a mapping of concepts there to concepts in GitLab. ### Sections @@ -217,7 +220,7 @@ and is meant to be a mapping of concepts there to concepts in GitLab. The agent section is used to define how a pipeline will be executed. For GitLab, we use the [GitLab Runner](../runners/README.md) to provide this capability. You can configure your own runners in Kubernetes or on any host, or take advantage of our shared runner fleet (note that the shared runner fleet is only available for GitLab.com users.) The link above will bring you to the documentation which will describe how to get -up and running quickly. We also support using [tags](../runners/README.md#using-tags) to direct different jobs +up and running quickly. We also support using [tags](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) to direct different jobs to different Runners (execution agents). The `agent` section also allows you to define which Docker images should be used for execution, for which we use diff --git a/doc/ci/junit_test_reports.md b/doc/ci/junit_test_reports.md index a77044e849d..aa0d40a4d06 100644 --- a/doc/ci/junit_test_reports.md +++ b/doc/ci/junit_test_reports.md @@ -1,10 +1,13 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # JUnit test reports -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/45318) in GitLab 11.2. Requires GitLab Runner 11.2 and above. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45318) in GitLab 11.2. Requires GitLab Runner 11.2 and above. ## Overview @@ -189,6 +192,20 @@ cpp: junit: report.xml ``` +#### CUnit + +[CUnit](https://cunity.gitlab.io/cunit/) can be made to produce [JUnit XML reports](https://cunity.gitlab.io/cunit/group__CI.html) automatically when run using its `CUnitCI.h` macros: + +```yaml +cunit: + stage: test + script: + - ./my-cunit-test + artifacts: + reports: + junit: ./my-cunit-test.xml +``` + ### .Net example The [JunitXML.TestLogger](https://www.nuget.org/packages/JunitXml.TestLogger/) NuGet @@ -215,17 +232,9 @@ Test: - ./**/*test-result.xml ``` -## Limitations - -Currently, the following tools might not work because their XML formats are unsupported in GitLab. - -|Case|Tool|Issue| -|---|---|---| -|`<testcase>` does not have `classname` attribute|ESlint, sass-lint|<https://gitlab.com/gitlab-org/gitlab-foss/issues/50964>| - ## Viewing JUnit test reports on GitLab -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/24792) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24792) in GitLab 12.5. If JUnit XML files are generated and uploaded as part of a pipeline, these reports can be viewed inside the pipelines details page. The **Tests** tab on this page will @@ -250,6 +259,9 @@ following command: ```ruby Feature.enable(:junit_pipeline_view) + +# Enable the feature for a specific project +Feature.enable(:junit_pipeline_view, Project.find(<your-project-id-here>)) ``` ## Viewing JUnit screenshots on GitLab diff --git a/doc/ci/large_repositories/index.md b/doc/ci/large_repositories/index.md index b4059fc252b..b4652050654 100644 --- a/doc/ci/large_repositories/index.md +++ b/doc/ci/large_repositories/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -114,6 +117,16 @@ For exact parameters accepted by for [`git clean`](https://git-scm.com/docs/git-clean). The available parameters are dependent on Git version. +## Git fetch extra flags + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4142) in GitLab Runner 13.1. + +[`GIT_FETCH_EXTRA_FLAGS`](../yaml/README.md#git-fetch-extra-flags) allows you +to modify `git fetch` behavior by passing extra flags. + +See the [`GIT_FETCH_EXTRA_FLAGS` documentation](../yaml/README.md#git-fetch-extra-flags) +for more information. + ## Fork-based workflow > Introduced in GitLab Runner 11.10. diff --git a/doc/ci/merge_request_pipelines/index.md b/doc/ci/merge_request_pipelines/index.md index a724bf416b6..7a4ca989fb6 100644 --- a/doc/ci/merge_request_pipelines/index.md +++ b/doc/ci/merge_request_pipelines/index.md @@ -1,11 +1,14 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, index last_update: 2019-07-03 --- # Pipelines for Merge Requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/15310) in GitLab 11.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/15310) in GitLab 11.6. In a [basic configuration](../pipelines/pipeline_architectures.md#basic-pipelines), GitLab runs a pipeline each time changes are pushed to a branch. @@ -143,7 +146,7 @@ test: Instead, you can use the [`$CI_COMMIT_REF_NAME` predefined environment -variable](../variables/predefined_variables.md#variables-reference) in +variable](../variables/predefined_variables.md) in combination with [`only:variables`](../yaml/README.md#onlyvariablesexceptvariables) to accomplish this behavior: diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md index c35a5d0a07e..a43085c8e7a 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md @@ -1,11 +1,14 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference last_update: 2019-07-03 --- # Pipelines for Merged Results **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7380) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7380) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. When you submit a merge request, you are requesting to merge changes from a source branch into a target branch. By default, the CI pipeline runs jobs @@ -44,7 +47,7 @@ To enable pipelines for merge results: - You must have maintainer [permissions](../../../user/permissions.md). - You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. - You must not be forking or using cross-repo workflows. To follow progress, - see [#11934](https://gitlab.com/gitlab-org/gitlab/issues/11934). + see [#11934](https://gitlab.com/gitlab-org/gitlab/-/issues/11934). - You must not be using [fast forward merges](../../../user/project/merge_requests/fast_forward_merge.md) yet. To follow progress, see [#58226](https://gitlab.com/gitlab-org/gitlab/-/issues/26996). @@ -56,7 +59,7 @@ To enable pipelines for merged results for your project: 1. [Configure your CI/CD configuration file](../index.md#configuring-pipelines-for-merge-requests) so that the pipeline or individual jobs run for merge requests. 1. Visit your project's **Settings > General** and expand **Merge requests**. -1. Check **Merge pipelines will try to validate the post-merge result prior to merging**. +1. Check **Enable merge trains and pipelines for merged results**. 1. Click **Save changes**. CAUTION: **Caution:** @@ -76,9 +79,9 @@ merge happens. For more information, read the [documentation on Merge Trains](merge_trains/index.md). -## Automatic pipeline cancelation +## Automatic pipeline cancellation -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12996) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12996) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. GitLab CI/CD can detect the presence of redundant pipelines, and will cancel them automatically in order to conserve CI resources. diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md index d921b75aa44..fc8e6368d72 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md @@ -1,12 +1,15 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference last_update: 2019-07-03 --- # Merge Trains **(PREMIUM)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9186) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.0. -> - [Squash and merge](../../../../user/project/merge_requests/squash_and_merge.md) support [introduced](https://gitlab.com/gitlab-org/gitlab/issues/13001) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9186) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.0. +> - [Squash and merge](../../../../user/project/merge_requests/squash_and_merge.md) support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13001) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. When [pipelines for merged results](../index.md#pipelines-for-merged-results-premium) are enabled, the pipeline jobs run as if the changes from your source branch have already @@ -79,7 +82,7 @@ To enable merge trains for your project: 1. [Configure your CI/CD configuration file](../../index.md#configuring-pipelines-for-merge-requests) so that the pipeline or individual jobs run for merge requests. 1. Visit your project's **Settings > General** and expand **Merge requests**. -1. Check **Merge pipelines will try to validate the post-merge result prior to merging**. +1. Check **Enable merge trains and pipelines for merged results**. 1. Click **Save changes**. CAUTION: **Caution:** @@ -164,7 +167,7 @@ To check the reason: [Merge When Pipeline Succeeds](../../../../user/project/merge_requests/merge_when_pipeline_succeeds.md) is currently unavailable when Merge Trains are enabled. -See [the related issue](https://gitlab.com/gitlab-org/gitlab/issues/12267) +See [the related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/12267) for more information. ### Merge Train Pipeline cannot be retried @@ -184,14 +187,14 @@ run a new successful pipeline before you can re-add a merge request to a merge t Merge trains ensure that each pipeline has succeeded before a merge happens, so you can clear the **Pipelines must succeed** check box and keep -**Merge pipelines will try to validate the post-merge result prior to merging** (merge trains) enabled. +**Enable merge trains and pipelines for merged results** (merge trains) enabled. If you want to keep the **Pipelines must succeed** option enabled along with Merge Trains, you can create a new pipeline for merged results when this error occurs by going to the **Pipelines** tab and clicking **Run pipeline**. Then click **Start/Add to merge train when pipeline succeeds**. -See [the related issue](https://gitlab.com/gitlab-org/gitlab/issues/35135) +See [the related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/35135) for more information. ### Merge Trains feature flag **(PREMIUM ONLY)** diff --git a/doc/ci/metrics_reports.md b/doc/ci/metrics_reports.md index f353aa2670f..67f9ad00c90 100644 --- a/doc/ci/metrics_reports.md +++ b/doc/ci/metrics_reports.md @@ -1,14 +1,17 @@ --- +stage: Verify +group: Testing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Metrics Reports **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9788) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. Requires GitLab Runner 11.10 and above. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9788) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. Requires GitLab Runner 11.10 and above. ## Overview -GitLab provides a lot of great reporting tools for [merge requests](../user/project/merge_requests/index.md) - [JUnit reports](junit_test_reports.md), [codequality](../user/project/merge_requests/code_quality.md), performance tests, etc. While JUnit is a great open framework for tests that "pass" or "fail", it is also important to see other types of metrics from a given change. +GitLab provides a lot of great reporting tools for [merge requests](../user/project/merge_requests/index.md) - [JUnit reports](junit_test_reports.md), [code quality](../user/project/merge_requests/code_quality.md), performance tests, etc. While JUnit is a great open framework for tests that "pass" or "fail", it is also important to see other types of metrics from a given change. You can configure your job to use custom Metrics Reports, and GitLab will display a report on the merge request so that it's easier and faster to identify changes without having to check the entire log. diff --git a/doc/ci/migration/circleci.md b/doc/ci/migration/circleci.md new file mode 100644 index 00000000000..f6868abc334 --- /dev/null +++ b/doc/ci/migration/circleci.md @@ -0,0 +1,332 @@ +--- +comments: false +type: index, howto +--- + +# Migrating from CircleCI + +If you are currently using CircleCI, you can migrate your CI/CD pipelines to [GitLab CI/CD](../introduction/index.md), +and start making use of all its powerful features. Check out our +[CircleCI vs GitLab](https://about.gitlab.com/devops-tools/circle-ci-vs-gitlab.html) +comparison to see what's different. + +We have collected several resources that you may find useful before starting to migrate. + +The [Quick Start Guide](../quick_start/README.md) is a good overview of how GitLab CI/CD works. You may also be interested in [Auto DevOps](../../topics/autodevops/index.md) which can be used to build, test, and deploy your applications with little to no configuration needed at all. + +For advanced CI/CD teams, [custom project templates](../../user/admin_area/custom_project_templates.md) can enable the reuse of pipeline configurations. + +If you have questions that are not answered here, the [GitLab community forum](https://forum.gitlab.com/) can be a great resource. + +## `config.yml` vs `gitlab-ci.yml` + +CircleCI's `config.yml` configuration file defines scripts, jobs, and workflows (known as "stages" in GitLab). In GitLab, a similar approach is used with a `.gitlab-ci.yml` file in the root directory of your repository. + +### Jobs + +In CircleCI, jobs are a collection of steps to perform a specific task. In GitLab, [jobs](../yaml/README.md#introduction) are also a fundamental element in the configuration file. The `checkout` parameter is not necessary in GitLab CI/CD as the repository is automatically fetched. + +CircleCI example job definition: + +```yaml +jobs: + job1: + steps: + - checkout + - run: "execute-script-for-job1" +``` + +Example of the same job definition in GitLab CI/CD: + +``` yaml +job1: + script: "execute-script-for-job1" +``` + +### Docker image definition + +CircleCI defines images at the job level, which is also supported by GitLab CI/CD. Additionally, GitLab CI/CD supports setting this globally to be used by all jobs that don't have `image` defined. + +CircleCI example image definition: + +```yaml +jobs: + job1: + docker: + - image: ruby:2.6 +``` + +Example of the same image definition in GitLab CI/CD: + +```yaml +job1: + image: ruby:2.6 +``` + +### Workflows + +CircleCI determines the run order for jobs with `workflows`. This is also used to determine concurrent, sequential, scheduled, or manual runs. The equivalent function in GitLab CI/CD is called [stages](../yaml/README.md#stages). Jobs on the same stage run in parallel, and only run after previous stages complete. Execution of the next stage is skipped when a job fails by default, but this can be allowed to continue even [after a failed job](../yaml/README.md#allow_failure). + +See [the Pipeline Architecture Overview](../pipelines/pipeline_architectures.md) for guidance on different types of pipelines that you can use. Pipelines can be tailored to meet your needs, such as for a large complex project or a monorepo with independent defined components. + +#### Parallel and sequential job execution + +The following examples show how jobs can run in parallel, or sequentially: + +1. `job1` and `job2` run in parallel (in the `build` stage for GitLab CI/CD). +1. `job3` runs only after `job1` and `job2` complete successfully (in the `test` stage). +1. `job4` runs only after `job3` completes successfully (in the `deploy` stage). + +CircleCI example with `workflows`: + +```yaml +version: 2 +jobs: + job1: + steps: + - checkout + - run: make build dependencies + job2: + steps: + - run: make build artifacts + job3: + steps: + - run: make test + job4: + steps: + - run: make deploy + +workflows: + version: 2 + jobs: + - job1 + - job2 + - job3: + requires: + - job1 + - job2 + - job4: + requires: + - job3 +``` + +Example of the same workflow as `stages` in GitLab CI/CD: + +```yaml +stages: + - build + - test + - deploy + +job 1: + stage: build + script: make build dependencies + +job 2: + stage: build + script: make build artifacts + +job3: + stage: test + script: make test + +job4: + stage: deploy + script: make deploy +``` + +#### Scheduled run + +GitLab CI/CD has an easy to use UI to [schedule pipelines](../pipelines/schedules.md). Also, [rules](../yaml/README.md#rules) can be used to determine if jobs should be included or excluded from a scheduled pipeline. + +CircleCI example of a scheduled workflow: + +```yaml +commit-workflow: + jobs: + - build +scheduled-workflow: + triggers: + - schedule: + cron: "0 1 * * *" + filters: + branches: + only: try-schedule-workflow + jobs: + - build +``` + +Example of the same scheduled pipeline using [`rules`](../yaml/README.md#rules) in GitLab CI/CD: + +```yaml +job1: + script: + - make build + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "try-schedule-workflow"' +``` + +After the pipeline configuration is saved, you configure the cron schedule in the [GitLab UI](../pipelines/schedules.md#configuring-pipeline-schedules), and can enable or disable schedules in the UI as well. + +#### Manual run + +CircleCI example of a manual workflow: + +```yaml +release-branch-workflow: + jobs: + - build + - testing: + requires: + - build + - deploy: + type: approval + requires: + - testing +``` + +Example of the same workflow using [`when: manual`](../yaml/README.md#whenmanual) in GitLab CI/CD: + +```yaml +deploy_prod: + stage: deploy + script: + - echo "Deploy to production server" + when: manual +``` + +### Filter job by branch + +[Rules](../yaml/README.md#rules) are a mechanism to determine if the job will or will not run for a specific branch. + +CircleCI example of a job filtered by branch: + +```yaml +jobs: + deploy: + branches: + only: + - master + - /rc-.*/ +``` + +Example of the same workflow using `rules` in GitLab CI/CD: + +```yaml +deploy_prod: + stage: deploy + script: + - echo "Deploy to production server" + rules: + - if: '$CI_COMMIT_BRANCH == "master"' +``` + +### Caching + +GitLab provides a caching mechanism to speed up build times for your jobs by reusing previously downloaded dependencies. It's important to know the different between [cache and artifacts](../caching/index.md#cache-vs-artifacts) to make the best use of these features. + +CircleCI example of a job using a cache: + +```yaml +jobs: + job1: + steps: + - restore_cache: + key: source-v1-< .Revision > + - checkout + - run: npm install + - save_cache: + key: source-v1-< .Revision > + paths: + - "node_modules" +``` + +Example of the same pipeline using `cache` in GitLab CI/CD: + +```yaml +image: node:latest + +# Cache modules in between jobs +cache: + key: $CI_COMMIT_REF_SLUG + paths: + - .npm/ + +before_script: + - npm ci --cache .npm --prefer-offline + +test_async: + script: + - node ./specs/start.js ./specs/async.spec.js +``` + +## Contexts and variables + +CircleCI provides [Contexts](https://circleci.com/docs/2.0/contexts/) to securely pass environment variables across project pipelines. In GitLab, a [Group](../../user/group/index.md) can be created to assemble related projects together. At the group level, [variables](../variables/README.md#group-level-environment-variables) can be stored outside the individual projects, and securely passed into pipelines across multiple projects. + +## Orbs + +There are two GitLab issues open addressing CircleCI Orbs and how GitLab can achieve similar functionality. + +- <https://gitlab.com/gitlab-com/Product/-/issues/1151> +- <https://gitlab.com/gitlab-org/gitlab/-/issues/195173> + +## Build environments + +CircleCI offers `executors` as the underlying technology to run a specific job. In GitLab, this is done by [Runners](https://docs.gitlab.com/runner/). + +The following environments are supported: + +Self-Managed Runners: + +- Linux +- Windows +- macOS + +GitLab.com Shared Runners: + +- Linux +- Windows +- [Planned: macOS](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/5720) + +### Machine and specific build environments + +[Tags](../yaml/README.md#tags) can be used to run jobs on different platforms, by telling GitLab which Runners should run the jobs. + +CircleCI example of a job running on a specific environment: + +```yaml +jobs: + ubuntuJob: + machine: + image: ubuntu-1604:201903-01 + steps: + - checkout + - run: echo "Hello, $USER!" + osxJob: + macos: + xcode: 11.3.0 + steps: + - checkout + - run: echo "Hello, $USER!" +``` + +Example of the same job using `tags` in GitLab CI/CD: + +```yaml +windows job: + stage: + - build + tags: + - windows + script: + - echo Hello, %USERNAME%! + +osx job: + stage: + - build + tags: + - osx + script: + - echo "Hello, $USER!" +``` diff --git a/doc/ci/multi_project_pipelines.md b/doc/ci/multi_project_pipelines.md index 2627f83c043..40d495a69f1 100644 --- a/doc/ci/multi_project_pipelines.md +++ b/doc/ci/multi_project_pipelines.md @@ -1,11 +1,14 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Multi-project pipelines > - [Introduced](https://about.gitlab.com/releases/2015/08/22/gitlab-7-14-released/#build-triggers-api-gitlab-ci) in GitLab 7.14, as Build Triggers. -> - [Made available](https://gitlab.com/gitlab-org/gitlab/issues/199224) in all tiers in GitLab 12.8. +> - [Made available](https://gitlab.com/gitlab-org/gitlab/-/issues/199224) in all tiers in GitLab 12.8. You can set up [GitLab CI/CD](README.md) across multiple projects, so that a pipeline in one project can trigger a pipeline in another project. @@ -35,7 +38,7 @@ With Multi-Project Pipelines you can visualize the entire pipeline, including al ## Multi-project pipeline visualization **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2121) in [GitLab Premium 9.3](https://about.gitlab.com/releases/2017/06/22/gitlab-9-3-released/#multi-project-pipeline-graphs). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2121) in [GitLab Premium 9.3](https://about.gitlab.com/releases/2017/06/22/gitlab-9-3-released/#multi-project-pipeline-graphs). When you configure GitLab CI/CD for your project, you can visualize the stages of your [jobs](pipelines/index.md#configure-a-pipeline) on a [pipeline graph](pipelines/index.md#visualize-pipelines). @@ -50,7 +53,7 @@ and when hovering or tapping (on touchscreen devices) they will expand and be sh ## Triggering multi-project pipelines through API > - Use of `CI_JOB_TOKEN` for multi-project pipelines was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2017) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.3. -> - Use of `CI_JOB_TOKEN` for multi-project pipelines was [made available](https://gitlab.com/gitlab-org/gitlab/issues/31573) in all tiers in GitLab 12.4. +> - Use of `CI_JOB_TOKEN` for multi-project pipelines was [made available](https://gitlab.com/gitlab-org/gitlab/-/issues/31573) in all tiers in GitLab 12.4. When you use the [`CI_JOB_TOKEN` to trigger pipelines](triggers/README.md#ci-job-token), GitLab recognizes the source of the job token, and thus internally ties these pipelines @@ -61,8 +64,8 @@ outbound connections for upstream and downstream pipeline dependencies. ## Creating multi-project pipelines from `.gitlab-ci.yml` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8. -> - [Made available](https://gitlab.com/gitlab-org/gitlab/issues/199224) in all tiers in 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8. +> - [Made available](https://gitlab.com/gitlab-org/gitlab/-/issues/199224) in all tiers in 12.8. ### Triggering a downstream pipeline using a bridge job @@ -129,7 +132,7 @@ Use: - The `project` keyword to specify the full path to a downstream project. - The `branch` keyword to specify the name of a branch in the project specified by `project`. - [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/10126), variable expansion is + [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/10126), variable expansion is supported. GitLab will use a commit that is currently on the HEAD of the branch when @@ -192,8 +195,8 @@ the ones defined in the upstream project will take precedence. ### Mirroring status from triggered pipeline -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11238) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/199224) to GitLab Core in 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11238) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/199224) to GitLab Core in 12.8. You can mirror the pipeline status from the triggered pipeline to the source bridge job by using `strategy: depend`. For example: @@ -240,7 +243,7 @@ Some features are not implemented yet. For example, support for environments. ## Trigger a pipeline when an upstream project is rebuilt -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9045) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9045) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. You can trigger a pipeline in your project whenever a pipeline finishes for a new tag in a different project: diff --git a/doc/ci/parent_child_pipelines.md b/doc/ci/parent_child_pipelines.md index e28adc2bc01..c936bd8f5fe 100644 --- a/doc/ci/parent_child_pipelines.md +++ b/doc/ci/parent_child_pipelines.md @@ -1,10 +1,13 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Parent-child pipelines -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/16094) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16094) in GitLab 12.7. As pipelines grow more complex, a few related problems start to emerge: @@ -131,7 +134,7 @@ microservice_a: ## Dynamic child pipelines -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/35632) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35632) in GitLab 12.9. Instead of running a child pipeline from a static YAML file, you can define a job that runs your own script to generate a YAML file, which is then [used to trigger a child pipeline](yaml/README.md#trigger-child-pipeline-with-generated-configuration-file). @@ -148,5 +151,5 @@ This is [resolved in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues ## Limitations A parent pipeline can trigger many child pipelines, but a child pipeline cannot trigger -further child pipelines. See the [related issue](https://gitlab.com/gitlab-org/gitlab/issues/29651) +further child pipelines. See the [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/29651) for discussion on possible future improvements. diff --git a/doc/ci/pipelines/img/code_coverage_graph_v13_1.png b/doc/ci/pipelines/img/code_coverage_graph_v13_1.png Binary files differnew file mode 100644 index 00000000000..da92a1b4a86 --- /dev/null +++ b/doc/ci/pipelines/img/code_coverage_graph_v13_1.png diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 731cb196eaa..48aaebc3456 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/ci/pipelines.html' type: reference --- @@ -88,7 +91,7 @@ Clicking a pipeline will bring you to the **Pipeline Details** page and show the jobs that were run for that pipeline. From here you can cancel a running pipeline, retry jobs on a failed pipeline, or [delete a pipeline](#delete-a-pipeline). -[Starting in GitLab 12.3](https://gitlab.com/gitlab-org/gitlab-foss/issues/50499), a link to the +[Starting in GitLab 12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/50499), a link to the latest pipeline for the last commit of a given branch is available at `/project/pipelines/[branch]/latest`. Also, `/project/pipelines/latest` will redirect you to the latest pipeline for the last commit on the project's default branch. @@ -98,6 +101,8 @@ you can filter the pipeline list by: - Trigger author - Branch name +- Status ([since GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/217617)) +- Tag ([since GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/217617)) ### Run a pipeline manually @@ -119,7 +124,7 @@ The pipeline will execute the jobs as configured. ### Run a pipeline by using a URL query string -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/24146) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24146) in GitLab 12.5. You can use a query string to pre-populate the **Run Pipeline** page. For example, the query string `.../pipelines/new?ref=my_branch&var[foo]=bar&file_var[file_foo]=file_bar` will pre-populate the @@ -179,7 +184,7 @@ This functionality is only available: ### Delete a pipeline -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/24851) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24851) in GitLab 12.7. Users with [owner permissions](../../user/permissions.md) in a project can delete a pipeline by clicking on the pipeline in the **CI/CD > Pipelines** to get to the **Pipeline Details** @@ -403,7 +408,7 @@ For example, if you start rolling out new code and: ### Expand and collapse job log sections -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/14664) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14664) in GitLab 12.0. Job logs are divided into sections that can be collapsed or expanded. Each section will display the duration. @@ -500,7 +505,7 @@ Stages in pipeline mini graphs are collapsible. Hover your mouse over them and c ### Pipeline success and duration charts > - Introduced in GitLab 3.1.1 as Commit Stats, and later renamed to Pipeline Charts. -> - [Renamed](https://gitlab.com/gitlab-org/gitlab/issues/38318) to CI / CD Analytics in GitLab 12.8. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/38318) to CI / CD Analytics in GitLab 12.8. GitLab tracks the history of your pipeline successes and failures, as well as how long each pipeline ran. To view this information, go to **Analytics > CI / CD Analytics**. diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index d4774016d9c..1d60f412e5e 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/job_artifacts.html' type: reference, howto --- @@ -103,7 +106,7 @@ combination thereof (`junit: [rspec.xml, test-results/TEST-*.xml]`). #### `artifacts:reports:dotenv` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17066) in GitLab 12.9. > - Requires GitLab Runner 11.5 and later. The `dotenv` report collects a set of environment variables as artifacts. @@ -122,7 +125,7 @@ There are a couple of limitations on top of the [original dotenv rules](https:// #### `artifacts:reports:cobertura` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3708) in GitLab 12.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3708) in GitLab 12.9. > - Requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 and above. The `cobertura` report collects [Cobertura coverage XML files](../../user/project/merge_requests/test_coverage_visualization.md). @@ -137,9 +140,10 @@ third party ports for other languages like JavaScript, Python, Ruby, and so on. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207528) in GitLab 13.0. > - Requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 and above. -The `terraform` report obtains a Terraform `tfplan.json` file. The collected Terraform +The `terraform` report obtains a Terraform `tfplan.json` file. [JQ processing required to remove credentials](../../user/infrastructure/index.md#output-terraform-plan-information-into-a-merge-request). The collected Terraform plan report will be uploaded to GitLab as an artifact and will be automatically shown -in merge requests. +in merge requests. For more information, see +[Output `terraform plan` information into a merge request](../../user/infrastructure/index.md#output-terraform-plan-information-into-a-merge-request). #### `artifacts:reports:codequality` **(STARTER)** @@ -164,6 +168,18 @@ The collected SAST report will be uploaded to GitLab as an artifact and will be in the merge requests and pipeline view. It's also used to provide data for security dashboards. +#### `artifacts:reports:secret_detection` **(ULTIMATE)** + +> - Introduced in GitLab 13.1. +> - Requires GitLab Runner 11.5 and above. + +The `secret-detection` report collects [detected secrets](../../user/application_security/secret_detection/index.md) +as artifacts. + +The collected Secret Detection report is uploaded to GitLab as an artifact and summarized +in the merge requests and pipeline view. It's also used to provide data for security +dashboards. + #### `artifacts:reports:dependency_scanning` **(ULTIMATE)** > - Introduced in GitLab 11.5. @@ -250,6 +266,17 @@ as artifacts. The collected Metrics report will be uploaded to GitLab as an artifact and will be automatically shown in merge requests. +#### `artifacts:reports:requirements` **(ULTIMATE)** + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2859) in GitLab 13.1. +> - Requires GitLab Runner 11.5 and above. + +The `requirements` report collects `requirements.json` files as artifacts. + +The collected Requirements report will be uploaded to GitLab as an artifact and +existing [requirements](../../user/project/requirements/index.md) will be +marked as Satisfied. + ## Browsing artifacts > - From GitLab 9.2, PDFs, images, videos, and other formats can be previewed directly in the job artifacts browser without the need to download them. diff --git a/doc/ci/pipelines/pipeline_architectures.md b/doc/ci/pipelines/pipeline_architectures.md index 4872e2595f9..9176444660d 100644 --- a/doc/ci/pipelines/pipeline_architectures.md +++ b/doc/ci/pipelines/pipeline_architectures.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/ci/pipelines/schedules.md b/doc/ci/pipelines/schedules.md index 0c0a613c628..e488179ffee 100644 --- a/doc/ci/pipelines/schedules.md +++ b/doc/ci/pipelines/schedules.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/schedules.html' type: reference, howto --- diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 0a859b5b68f..2c9d4ccf2c4 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/settings.html' type: reference, howto --- @@ -59,7 +62,7 @@ if the job surpasses the threshold, it is marked as failed. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17221) in GitLab 10.7. Project defined timeout (either specific timeout set by user or the default -60 minutes timeout) may be [overridden on Runner level](../runners/README.md#setting-maximum-job-timeout-for-a-runner). +60 minutes timeout) may be [overridden on Runner level](../runners/README.md#set-maximum-job-timeout-for-a-runner). ## Maximum artifacts size **(CORE ONLY)** @@ -69,7 +72,7 @@ For information about setting a maximum artifact size for a project, see ## Custom CI configuration path > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12509) in GitLab 9.4. -> - [Support for external `.gitlab-ci.yml` locations](https://gitlab.com/gitlab-org/gitlab/issues/14376) introduced in GitLab 12.6. +> - [Support for external `.gitlab-ci.yml` locations](https://gitlab.com/gitlab-org/gitlab/-/issues/14376) introduced in GitLab 12.6. By default we look for the `.gitlab-ci.yml` file in the project's root directory. If needed, you can specify an alternate path and file name, including locations outside the project. @@ -118,7 +121,8 @@ job log using a regular expression. In the pipelines settings, search for the  Leave blank if you want to disable it or enter a Ruby regular expression. You -can use <https://rubular.com> to test your regex. +can use <https://rubular.com> to test your regex. The regex returns the **last** +match found in the output. If the pipeline succeeds, the coverage is shown in the merge request widget and in the jobs table. @@ -130,15 +134,18 @@ in the jobs table. A few examples of known coverage tools for a variety of languages can be found in the pipelines settings page. -### Download test coverage history +### Code Coverage history -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209121) in GitLab 12.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209121) the ability to download a `.csv` in GitLab 12.10. +> - [Graph introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33743) in GitLab 13.1. If you want to see the evolution of your project code coverage over time, -you can download a CSV file with this data. From your project: +you can view a graph or download a CSV file with this data. From your project: -1. Go to **{chart}** **Project Analytics > Repository**. -1. Click **Download raw data (.csv)** +1. Go to **{chart}** **Project Analytics > Repository** to see the historic data for each job listed in the dropdown above the graph. +1. If you want a CSV file of that data, click **Download raw data (.csv)** + + ### Removing color codes @@ -205,9 +212,11 @@ you can enable this in the project settings: 1. Check the **Auto-cancel redundant, pending pipelines** checkbox. 1. Click **Save changes**. +Note that only jobs with [interruptible](../yaml/README.md#interruptible) set to `true` will be cancelled. + ## Skip outdated deployment jobs -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25276) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25276) in GitLab 12.9. Your project may have multiple concurrent deployment jobs that are scheduled to run within the same time frame. @@ -224,6 +233,8 @@ To avoid this scenario: The pending deployment jobs will be skipped. +For more information, see [Deployment safety](../environments/deployment_safety.md). + ## Pipeline Badges In the pipelines settings page you can find pipeline status and test coverage @@ -289,7 +300,7 @@ https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?st #### Flat square -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/30120) in GitLab 11.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30120) in GitLab 11.8. ```plaintext https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?style=flat-square @@ -297,6 +308,16 @@ https://example.gitlab.com/<namespace>/<project>/badges/<branch>/coverage.svg?st  +### Custom badge text + +The text for a badge can be customized. This can be useful to differentiate between multiple coverage jobs that run in the same pipeline. Customize the badge text and width by adding the `key_text=custom_text` and `key_width=custom_key_width` parameters to the URL: + +```plaintext +https://gitlab.com/gitlab-org/gitlab-foss/badges/master/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=100 +``` + + + ## Environment Variables [Environment variables](../variables/README.md#gitlab-cicd-environment-variables) can be set in an environment to be available to a runner. diff --git a/doc/ci/quick_start/README.md b/doc/ci/quick_start/README.md index a14cfcfa9ed..25469570603 100644 --- a/doc/ci/quick_start/README.md +++ b/doc/ci/quick_start/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -165,7 +168,7 @@ The next step is to configure a Runner so that it picks the pending jobs. ## Configuring a Runner In GitLab, Runners run the jobs that you define in `.gitlab-ci.yml`. A Runner -can be a virtual machine, a VPS, a bare-metal machine, a docker container or +can be a virtual machine, a VPS, a bare-metal machine, a Docker container or even a cluster of containers. GitLab and the Runners communicate through an API, so the only requirement is that the Runner's machine has network access to the GitLab server. @@ -184,7 +187,7 @@ can be found at <https://docs.gitlab.com/runner/>. In order to have a functional Runner you need to follow two steps: 1. [Install it](https://docs.gitlab.com/runner/install/) -1. [Configure it](../runners/README.md#registering-a-specific-runner) +1. [Configure it](https://docs.gitlab.com/runner/configuration/) Follow the links above to set up your own Runner or use a Shared Runner as described in the next section. diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index ad0a4270f43..67fac70c8de 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -7,7 +7,7 @@ type: reference # Review Apps -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/21971) in GitLab 8.12. Further additions were made in GitLab 8.13 and 8.14. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/21971) in GitLab 8.12. Further additions were made in GitLab 8.13 and 8.14. > - Inspired by [Heroku's Review Apps](https://devcenter.heroku.com/articles/github-integration-review-apps), which itself was inspired by [Fourchette](https://github.com/rainforestapp/fourchette). Review Apps is a collaboration tool that takes the hard work out of providing an environment to showcase product changes. @@ -65,7 +65,7 @@ The process of configuring Review Apps is as follows: ### Enable Review Apps button -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/118844) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118844) in GitLab 12.8. When configuring Review Apps for a project, you need to add a new job to `.gitlab-ci.yml`, as mentioned above. To facilitate this and if you are using Kubernetes, you can click @@ -95,8 +95,11 @@ The following are example projects that demonstrate Review App configuration: - [NGINX](https://gitlab.com/gitlab-examples/review-apps-nginx). - [OpenShift](https://gitlab.com/gitlab-examples/review-apps-openshift). -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -See also the video [Demo: Cloud Native Development with GitLab](https://www.youtube.com/watch?v=jfIyQEwrocw), which includes a Review Apps example. +Other examples of Review Apps: + +- <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +[Cloud Native Development with GitLab](https://www.youtube.com/watch?v=jfIyQEwrocw). +- [Review Apps for Android](https://about.gitlab.com/blog/2020/05/06/how-to-create-review-apps-for-android-with-gitlab-fastlane-and-appetize-dot-io/). ## Route Maps @@ -185,7 +188,7 @@ Once you have the route mapping set up, it will take effect in the following loc ## Visual Reviews **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10761) in GitLab Starter 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10761) in GitLab Starter 12.0. With Visual Reviews, you can provide a feedback form to your Review Apps so that reviewers can post comments directly from the app back to the merge request @@ -258,7 +261,7 @@ Then, when your app is deployed via GitLab CI/CD, those variables should get replaced with their real values. NOTE: **Note:** -Future enhancements [are planned](https://gitlab.com/gitlab-org/gitlab/issues/11322) +Future enhancements [are planned](https://gitlab.com/gitlab-org/gitlab/-/issues/11322) to make this process even easier. ### Determining merge request ID diff --git a/doc/ci/runners/README.md b/doc/ci/runners/README.md index deefd2f0e73..86fabd5d54f 100644 --- a/doc/ci/runners/README.md +++ b/doc/ci/runners/README.md @@ -1,190 +1,59 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Configuring GitLab Runners In GitLab CI/CD, Runners run the code defined in [`.gitlab-ci.yml`](../yaml/README.md). -They are isolated (virtual) machines that pick up jobs through the coordinator -API of GitLab CI/CD. - -A Runner can be specific to a certain project or serve any project -in GitLab CI/CD. A Runner that serves all projects is called a shared Runner. - -Ideally, the GitLab Runner should not be installed on the same machine as GitLab. -Read the [requirements documentation](../../install/requirements.md#gitlab-runner) -for more information. - -## Shared, specific and group Runners - -After [installing the Runner](https://docs.gitlab.com/runner/install/), you can either register it as shared or -specific. You can only register a shared Runner if you have admin access to -the GitLab instance. The main differences between a shared and a specific Runner -are: - -- **Shared Runners** are useful for jobs that have similar requirements, - between multiple projects. Rather than having multiple Runners idling for - many projects, you can have a single or a small number of Runners that handle - multiple projects. This makes it easier to maintain and update them. - Shared Runners process jobs using a [fair usage queue](#how-shared-runners-pick-jobs). - In contrast to specific Runners that use a FIFO queue, this prevents - cases where projects create hundreds of jobs which can lead to eating all - available shared Runners resources. -- **Specific Runners** are useful for jobs that have special requirements or for - projects with a specific demand. If a job has certain requirements, you can set - up the specific Runner with this in mind, while not having to do this for all - Runners. For example, if you want to deploy a certain project, you can set up - a specific Runner to have the right credentials for this. The [usage of tags](#using-tags) - may be useful in this case. Specific Runners process jobs using a [FIFO](https://en.wikipedia.org/wiki/FIFO_(computing_and_electronics)) queue. -- **Group Runners** are useful when you have multiple projects under one group - and would like all projects to have access to a set of Runners. Group Runners - process jobs using a [FIFO](https://en.wikipedia.org/wiki/FIFO_(computing_and_electronics)) queue. - -A Runner that is specific only runs for the specified project(s). A shared Runner -can run jobs for every project that has enabled the option **Allow shared Runners** -under **Settings > CI/CD**. - -Projects with high demand of CI activity can also benefit from using specific -Runners. By having dedicated Runners you are guaranteed that the Runner is not -being held up by another project's jobs. - -You can set up a specific Runner to be used by multiple projects. The difference -with a shared Runner is that you have to enable each project explicitly for -the Runner to be able to run its jobs. +A GitLab Runner is a lightweight, highly-scalable agent that picks up a CI job through +the coordinator API of GitLab CI/CD, runs the job, and sends the result back to the GitLab instance. -Specific Runners do not get shared with forked projects automatically. -A fork does copy the CI settings (jobs, allow shared, etc) of the cloned -repository. - -## Registering a shared Runner - -You can only register a shared Runner if you are an admin of the GitLab instance. - -1. Grab the shared-Runner token on the `admin/runners` page - -  - -1. [Register the Runner](https://docs.gitlab.com/runner/register/) - -Shared Runners are enabled by default as of GitLab 8.2, but can be disabled -with the **Disable shared Runners** button which is present under each project's -**Settings > CI/CD** page. Previous versions of GitLab defaulted shared -Runners to disabled. - -## Registering a specific Runner - -Registering a specific Runner can be done in two ways: - -1. Creating a Runner with the project registration token -1. Converting a shared Runner into a specific Runner (one-way, admin only) - -### Registering a specific Runner with a project registration token - -To create a specific Runner without having admin rights to the GitLab instance, -visit the project you want to make the Runner work for in GitLab: - -1. Go to **Settings > CI/CD** to obtain the token -1. [Register the Runner](https://docs.gitlab.com/runner/register/) - -## Registering a group Runner - -Creating a group Runner requires Owner permissions for the group. To create a -group Runner visit the group you want to make the Runner work for in GitLab: - -1. Go to **Settings > CI/CD** to obtain the token -1. [Register the Runner](https://docs.gitlab.com/runner/register/) - -### Making an existing shared Runner specific - -If you are an admin on your GitLab instance, you can turn any shared Runner into -a specific one, but not the other way around. Keep in mind that this is a one -way transition. - -1. Go to the Runners in the **Admin Area > Overview > Runners** (`/admin/runners`) - and find your Runner -1. Enable any projects under **Restrict projects for this Runner** to be used - with the Runner - -From now on, the shared Runner will be specific to those projects. +Runners are created by an administrator and are visible in the GitLab UI. +Runners can be specific to certain projects or available to all projects. -## Locking a specific Runner from being enabled for other projects +## Types of Runners -You can configure a Runner to assign it exclusively to a project. When a -Runner is locked this way, it can no longer be enabled for other projects. -This setting can be enabled the first time you [register a Runner](https://docs.gitlab.com/runner/register/) and -can be changed afterwards under each Runner's settings. +There are three types of Runners: -To lock/unlock a Runner: +- [Shared](#shared-runners) (for all projects) +- [Group](#group-runners) (for all projects in a group) +- [Specific](#specific-runners) (for specific projects) -1. Visit your project's **Settings > CI/CD** -1. Find the Runner you wish to lock/unlock and make sure it's enabled -1. Click the pencil button -1. Check the **Lock to current projects** option -1. Click **Save changes** for the changes to take effect +If you are running self-managed GitLab, you can create your own Runners. -## Assigning a Runner to another project +If you are using GitLab.com, you can use the shared Runners provided by GitLab or +create your own group or specific Runners. -If you are an Owner on a project where a specific Runner is assigned to, and the -Runner is not [locked only to that project](#locking-a-specific-runner-from-being-enabled-for-other-projects), -you can enable the Runner also on any other project where you have Owner permissions. - -To enable/disable a Runner in your project: - -1. Visit your project's **Settings > CI/CD** -1. Find the Runner you wish to enable/disable -1. Click **Enable for this project** or **Disable for this project** - -> **Note**: -Consider that if you don't lock your specific Runner to a specific project, any -user with Maintainer role in you project can assign your Runner to another arbitrary -project without requiring your authorization, so use it with caution. - -CAUTION: **Caution:** -Never add a private Runner that you're using in your private projects to a -project that you share with other people. - -CAUTION: **Caution:** -Never use a Runner from a project which has multiple maintainers in your -private project. - -An admin can enable/disable a specific Runner for projects: - -1. Navigate to **Admin > Runners** -1. Find the Runner you wish to enable/disable -1. Click edit on the Runner -1. Click **Enable** or **Disable** on the project - -## Protected Runners - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13194) in GitLab 10.0. +### Shared Runners -You can protect Runners from revealing sensitive information. -Whenever a Runner is protected, the Runner picks only jobs created on -[protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md), and ignores other jobs. +*Shared Runners* are available to every project in a GitLab instance. -To protect/unprotect Runners: +Use shared Runners when you have multiple jobs with similar requirements. Rather than +having multiple Runners idling for many projects, you can have a few Runners that handle +multiple projects. -1. Visit your project's **Settings > CI/CD** -1. Find a Runner you want to protect/unprotect and make sure it's enabled -1. Click the pencil button besides the Runner name -1. Check the **Protected** option -1. Click **Save changes** for the changes to take effect +If you are using a self-managed instance of GitLab, your administrator can create +shared Runners and configure them to use the +[executor](https://docs.gitlab.com/runner/executors/README.html) you want. - +If you are using GitLab.com, you can select from a list of +[shared Runners that GitLab maintains](../../user/gitlab_com/index.md#shared-runners). -## Manually clearing the Runners cache +#### How shared Runners pick jobs -Read [clearing the cache](../caching/index.md#clearing-the-cache). +Shared Runners process jobs by using a fair usage queue. This queue prevents +projects from creating hundreds of jobs and using all available +shared Runner resources. -## How shared Runners pick jobs - -Shared Runners abide to a process queue we call fair usage. The fair usage -algorithm tries to assign jobs to shared Runners from projects that have the -lowest number of jobs currently running on shared Runners. +The fair usage queue algorithm assigns jobs based on the projects that have the +fewest number of jobs already running on shared Runners. **Example 1** -We have following jobs in queue: +If these jobs are in the queue: - Job 1 for Project 1 - Job 2 for Project 1 @@ -193,20 +62,20 @@ We have following jobs in queue: - Job 5 for Project 2 - Job 6 for Project 3 -With the fair usage algorithm jobs are assigned in following order: +The fair usage algorithm assigns jobs in this order: -1. Job 1 is chosen first, because it has the lowest job number from projects with no running jobs (that is, all projects) -1. Job 4 is next, because 4 is now the lowest job number from projects with no running jobs (Project 1 has a job running) -1. Job 6 is next, because 6 is now the lowest job number from projects with no running jobs (Projects 1 and 2 have jobs running) -1. Job 2 is next, because, of projects with the lowest number of jobs running (each has 1), it is the lowest job number +1. Job 1 is chosen first, because it has the lowest job number from projects with no running jobs (that is, all projects). +1. Job 4 is next, because 4 is now the lowest job number from projects with no running jobs (Project 1 has a job running). +1. Job 6 is next, because 6 is now the lowest job number from projects with no running jobs (Projects 1 and 2 have jobs running). +1. Job 2 is next, because, of projects with the lowest number of jobs running (each has 1), it is the lowest job number. 1. Job 5 is next, because Project 1 now has 2 jobs running and Job 5 is the lowest remaining job number between Projects 2 and 3. -1. Lastly we choose Job 3... because it's the only job left +1. Finally is Job 3... because it's the only job left. --- **Example 2** -We have following jobs in queue: +If these jobs are in the queue: - Job 1 for project 1 - Job 2 for project 1 @@ -215,103 +84,129 @@ We have following jobs in queue: - Job 5 for project 2 - Job 6 for project 3 -With the fair usage algorithm jobs are assigned in following order: +The fair usage algorithm assigns jobs in this order: -1. Job 1 is chosen first, because it has the lowest job number from projects with no running jobs (that is, all projects) -1. We finish job 1 -1. Job 2 is next, because, having finished Job 1, all projects have 0 jobs running again, and 2 is the lowest available job number -1. Job 4 is next, because with Project 1 running a job, 4 is the lowest number from projects running no jobs (Projects 2 and 3) -1. We finish job 4 -1. Job 5 is next, because having finished Job 4, Project 2 has no jobs running again -1. Job 6 is next, because Project 3 is the only project left with no running jobs -1. Lastly we choose Job 3... because, again, it's the only job left (who says 1 is the loneliest number?) +1. Job 1 is chosen first, because it has the lowest job number from projects with no running jobs (that is, all projects). +1. We finish job 1. +1. Job 2 is next, because, having finished Job 1, all projects have 0 jobs running again, and 2 is the lowest available job number. +1. Job 4 is next, because with Project 1 running a job, 4 is the lowest number from projects running no jobs (Projects 2 and 3). +1. We finish job 4. +1. Job 5 is next, because having finished Job 4, Project 2 has no jobs running again. +1. Job 6 is next, because Project 3 is the only project left with no running jobs. +1. Lastly we choose Job 3... because, again, it's the only job left. -## Using shared Runners effectively +#### Enable a shared Runner -If you are planning to use shared Runners, there are several things you -should keep in mind. +By default, all projects can use shared Runners, and they are enabled by default. -### Using tags +However, you can enable or disable shared Runners for individual projects. -You must set up a Runner to be able to run all the different types of jobs -that it may encounter on the projects it's shared over. This would be -problematic for large amounts of projects, if it wasn't for tags. +To enable or disable a shared Runner: -By tagging a Runner for the types of jobs it can handle, you can make sure -shared Runners will [only run the jobs they are equipped to run](../yaml/README.md#tags). +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Click **Allow shared Runners** or **Disable shared Runners**. -For instance, at GitLab we have Runners tagged with "rails" if they contain -the appropriate dependencies to run Rails test suites. +### Group Runners -### Allowing Runners with tags to pick jobs without tags +Use *Group Runners* when you want all projects in a group +to have access to a set of Runners. -When you [register a Runner](https://docs.gitlab.com/runner/register/), its default behavior is to **only pick** -[tagged jobs](../yaml/README.md#tags). +Group Runners process jobs by using a first in, first out ([FIFO](https://en.wikipedia.org/wiki/FIFO_(computing_and_electronics))) queue. -NOTE: **Note:** -Owner [permissions](../../user/permissions.md) are required to change the -Runner settings. +#### Create a group Runner -To make a Runner pick untagged jobs: +You can create a group Runner for your self-managed GitLab instance or for GitLab.com. +You must have [Owner permissions](../../user/permissions.md#group-members-permissions) for the group. -1. Visit your project's **Settings > CI/CD > Runners**. -1. Find the Runner you want to pick untagged jobs and make sure it's enabled. -1. Click the pencil button. -1. Check the **Run untagged jobs** option. -1. Click the **Save changes** button for the changes to take effect. +To create a group Runner: + +1. [Install Runner](https://docs.gitlab.com/runner/install/). +1. Go to the group you want to make the Runner work for. +1. Go to **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Note the URL and token. +1. [Register the Runner](https://docs.gitlab.com/runner/register/). + +#### Pause or remove a group Runner + +You can pause or remove a group Runner. +You must have [Owner permissions](../../user/permissions.md#group-members-permissions) for the group. + +1. Go to the group you want to remove or pause the Runner for. +1. Go to **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Click **Pause** or **Remove Runner**. +1. On the confirmation dialog, click **OK**. + +### Specific Runners + +Use *Specific Runners* when you want to use Runners for specific projects. For example, +when you have: + +- Jobs with specific requirements, like a deploy job that requires credentials. +- Projects with a lot of CI activity that can benefit from being separate from other Runners. + +You can set up a specific Runner to be used by multiple projects. Specific Runners +must be enabled for each project explicitly. + +Specific Runners process jobs by using a first in, first out ([FIFO](https://en.wikipedia.org/wiki/FIFO_(computing_and_electronics))) queue. NOTE: **Note:** -The Runner tags list can not be empty when it's not allowed to pick untagged jobs. +Specific Runners do not get shared with forked projects automatically. +A fork *does* copy the CI / CD settings of the cloned repository. -Below are some example scenarios of different variations. +#### Create a specific Runner -#### Runner runs only tagged jobs +You can create a specific Runner for your self-managed GitLab instance or for GitLab.com. +You must have [Owner permissions](../../user/permissions.md#project-members-permissions) for the project. -The following examples illustrate the potential impact of the Runner being set -to run only tagged jobs. +To create a specific Runner: -Example 1: +1. [Install Runner](https://docs.gitlab.com/runner/install/). +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Note the URL and token. +1. [Register the Runner](https://docs.gitlab.com/runner/register/). -1. The Runner is configured to run only tagged jobs and has the `docker` tag. -1. A job that has a `hello` tag is executed and stuck. +#### Enable a specific Runner for a specific project -Example 2: +A specific Runner is available in the project it was created for. An administrator can +enable a specific Runner to apply to additional projects. -1. The Runner is configured to run only tagged jobs and has the `docker` tag. -1. A job that has a `docker` tag is executed and run. +- You must have Owner permissions for the project. +- The specific Runner must not be [locked](#prevent-a-specific-runner-from-being-enabled-for-other-projects). -Example 3: +To enable or disable a specific Runner for a project: -1. The Runner is configured to run only tagged jobs and has the `docker` tag. -1. A job that has no tags defined is executed and stuck. +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Click **Enable for this project** or **Disable for this project**. -#### Runner is allowed to run untagged jobs +#### Prevent a specific Runner from being enabled for other projects -The following examples illustrate the potential impact of the Runner being set -to run tagged and untagged jobs. +You can configure a specific Runner so it is "locked" and cannot be enabled for other projects. +This setting can be enabled when you first [register a Runner](https://docs.gitlab.com/runner/register/), +but can also be changed later. -Example 1: +To lock or unlock a Runner: -1. The Runner is configured to run untagged jobs and has the `docker` tag. -1. A job that has no tags defined is executed and run. -1. A second job that has a `docker` tag defined is executed and run. +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Find the Runner you want to lock or unlock. Make sure it's enabled. +1. Click the pencil button. +1. Check the **Lock to current projects** option. +1. Click **Save changes**. -Example 2: +## Manually clear the Runner cache -1. The Runner is configured to run untagged jobs and has no tags defined. -1. A job that has no tags defined is executed and run. -1. A second job that has a `docker` tag defined is stuck. +Read [clearing the cache](../caching/index.md#clearing-the-cache). + +## Set maximum job timeout for a Runner -### Setting maximum job timeout for a Runner +For each Runner, you can specify a *maximum job timeout*. This timeout, +if smaller than the [project defined timeout](../pipelines/settings.md#timeout), takes precedence. -For each Runner you can specify a _maximum job timeout_. Such timeout, -if smaller than [project defined timeout](../pipelines/settings.md#timeout), will take the precedence. This -feature can be used to prevent Shared Runner from being appropriated -by a project by setting a ridiculous big timeout (for example, one week). +This feature can be used to prevent your shared Runner from being overwhelmed +by a project that has jobs with a long timeout (for example, one week). -When not configured, Runner will not override project timeout. +When not configured, Runners will not override the project timeout. -How this feature will work: +How this feature works: **Example 1 - Runner timeout bigger than project timeout** @@ -334,7 +229,7 @@ How this feature will work: 1. You start a job 1. The job, if running longer, will be timed out after **30 minutes** -### Be careful with sensitive information +## Be careful with sensitive information With some [Runner Executors](https://docs.gitlab.com/runner/executors/README.html), if you can run a job on the Runner, you can get full access to the file system, @@ -349,6 +244,25 @@ The above is easily avoided by restricting the usage of shared Runners on large public GitLab instances, controlling access to your GitLab instance, and using more secure [Runner Executors](https://docs.gitlab.com/runner/executors/README.html). +### Prevent Runners from revealing sensitive information + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13194) in GitLab 10.0. + +You can protect Runners so they don't reveal sensitive information. +When a Runner is protected, the Runner picks jobs created on +[protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md) only, +and ignores other jobs. + +To protect or unprotect a Runner: + +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Find the Runner you want to protect or unprotect. Make sure it's enabled. +1. Click the pencil button. +1. Check the **Protected** option. +1. Click **Save changes**. + + + ### Forks Whenever a project is forked, it copies the settings of the jobs that relate @@ -356,23 +270,21 @@ to it. This means that if you have shared Runners set up for a project and someone forks that project, the shared Runners will also serve jobs of this project. -## Attack vectors in Runners +### Attack vectors in Runners Mentioned briefly earlier, but the following things of Runners can be exploited. We're always looking for contributions that can mitigate these [Security Considerations](https://docs.gitlab.com/runner/security/). -### Resetting the registration token for a Project +### Reset the Runner registration token for a project -If you think that registration token for a Project was revealed, you should -reset them. It's recommended because such a token can be used to register another -Runner to the Project. It may then be used to obtain the values of secret -variables or clone the project code, that normally may be unavailable for the -attacker. +If you think that a registration token for a project was revealed, you should +reset it. A token can be used to register another Runner for the project. That new Runner +may then be used to obtain the values of secret variables or to clone project code. To reset the token: -1. Go to **Settings > CI/CD** for a specified Project. +1. Go to the project's **{settings}** **Settings > CI/CD**. 1. Expand the **General pipelines settings** section. 1. Find the **Runner token** form field and click the **Reveal value** button. 1. Delete the value and save the form. @@ -384,7 +296,7 @@ any new Runners to the project. If you are using any tools to provision and register new Runners, the tokens used in those tools should be updated to reflect the value of the new token. -## Determining the IP address of a Runner +## Determine the IP address of a Runner > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17286) in GitLab 10.6. @@ -397,22 +309,88 @@ automatically updated in GitLab. The IP address for shared Runners and specific Runners can be found in different places. -### Shared Runners +### Determine the IP address of a shared Runner To view the IP address of a shared Runner you must have admin access to the GitLab instance. To determine this: -1. Visit **Admin Area > Overview > Runners** -1. Look for the Runner in the table and you should see a column for "IP Address" +1. Visit **{admin}** **Admin Area > Overview > Runners**. +1. Look for the Runner in the table and you should see a column for **IP Address**.  -### Specific Runners +### Determine the IP address of a specific Runner -You can find the IP address of a Runner for a specific project by: +To can find the IP address of a Runner for a specific project, +you must have Owner [permissions](../../user/permissions.md#project-members-permissions) for the project. -1. Visit your project's **Settings > CI/CD** -1. Find the Runner and click on it's ID which links you to the details page -1. On the details page you should see a row for "IP Address" +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. On the details page you should see a row for **IP Address**.  + +## Use tags to limit the number of jobs using the Runner + +You must set up a Runner to be able to run all the different types of jobs +that it may encounter on the projects it's shared over. This would be +problematic for large amounts of projects, if it weren't for tags. + +By tagging a Runner for the types of jobs it can handle, you can make sure +shared Runners will [only run the jobs they are equipped to run](../yaml/README.md#tags). + +For instance, at GitLab we have Runners tagged with `rails` if they contain +the appropriate dependencies to run Rails test suites. + +When you [register a Runner](https://docs.gitlab.com/runner/register/), its default behavior is to **only pick** +[tagged jobs](../yaml/README.md#tags). +To change this, you must have Owner [permissions](../../user/permissions.md#project-members-permissions) for the project. + +To make a Runner pick untagged jobs: + +1. Go to the project's **{settings}** **Settings > CI/CD** and expand the **Runners** section. +1. Find the Runner you want to pick untagged jobs and make sure it's enabled. +1. Click the pencil button. +1. Check the **Run untagged jobs** option. +1. Click the **Save changes** button for the changes to take effect. + +NOTE: **Note:** +The Runner tags list can not be empty when it's not allowed to pick untagged jobs. + +Below are some example scenarios of different variations. + +### Runner runs only tagged jobs + +The following examples illustrate the potential impact of the Runner being set +to run only tagged jobs. + +Example 1: + +1. The Runner is configured to run only tagged jobs and has the `docker` tag. +1. A job that has a `hello` tag is executed and stuck. + +Example 2: + +1. The Runner is configured to run only tagged jobs and has the `docker` tag. +1. A job that has a `docker` tag is executed and run. + +Example 3: + +1. The Runner is configured to run only tagged jobs and has the `docker` tag. +1. A job that has no tags defined is executed and stuck. + +### Runner is allowed to run untagged jobs + +The following examples illustrate the potential impact of the Runner being set +to run tagged and untagged jobs. + +Example 1: + +1. The Runner is configured to run untagged jobs and has the `docker` tag. +1. A job that has no tags defined is executed and run. +1. A second job that has a `docker` tag defined is executed and run. + +Example 2: + +1. The Runner is configured to run untagged jobs and has no tags defined. +1. A job that has no tags defined is executed and run. +1. A second job that has a `docker` tag defined is stuck. diff --git a/doc/ci/services/README.md b/doc/ci/services/README.md index 7fe12eb53e7..fda453b7aa0 100644 --- a/doc/ci/services/README.md +++ b/doc/ci/services/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false type: index --- diff --git a/doc/ci/services/mysql.md b/doc/ci/services/mysql.md index dcfd863709e..1b017111d22 100644 --- a/doc/ci/services/mysql.md +++ b/doc/ci/services/mysql.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -43,7 +46,7 @@ Database: <your_mysql_database> If you are wondering why we used `mysql` for the `Host`, read more at [How services are linked to the job](../docker/using_docker_images.md#how-services-are-linked-to-the-job). -You can also use any other docker image available on [Docker Hub](https://hub.docker.com/_/mysql/). +You can also use any other Docker image available on [Docker Hub](https://hub.docker.com/_/mysql/). For example, to use MySQL 5.5 the service becomes `mysql:5.5`. The `mysql` image can accept some environment variables. For more details diff --git a/doc/ci/services/postgres.md b/doc/ci/services/postgres.md index 2f92bd969ff..aadbce5a50a 100644 --- a/doc/ci/services/postgres.md +++ b/doc/ci/services/postgres.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -45,7 +48,7 @@ Database: nice_marmot If you are wondering why we used `postgres` for the `Host`, read more at [How services are linked to the job](../docker/using_docker_images.md#how-services-are-linked-to-the-job). -You can also use any other docker image available on [Docker Hub](https://hub.docker.com/_/postgres). +You can also use any other Docker image available on [Docker Hub](https://hub.docker.com/_/postgres). For example, to use PostgreSQL 9.3 the service becomes `postgres:9.3`. The `postgres` image can accept some environment variables. For more details diff --git a/doc/ci/services/redis.md b/doc/ci/services/redis.md index f22ee87a9d3..193f78f240b 100644 --- a/doc/ci/services/redis.md +++ b/doc/ci/services/redis.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -30,7 +33,7 @@ Host: redis And that's it. Redis will now be available to be used within your testing framework. -You can also use any other docker image available on [Docker Hub](https://hub.docker.com/_/redis). +You can also use any other Docker image available on [Docker Hub](https://hub.docker.com/_/redis). For example, to use Redis 2.8 the service becomes `redis:2.8`. ## Use Redis with the Shell executor diff --git a/doc/ci/ssh_keys/README.md b/doc/ci/ssh_keys/README.md index 16e87e92a07..4ad4758b281 100644 --- a/doc/ci/ssh_keys/README.md +++ b/doc/ci/ssh_keys/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers last_updated: 2017-12-13 type: tutorial --- diff --git a/doc/ci/triggers/README.md b/doc/ci/triggers/README.md index db3d2352f09..c6ac87ef888 100644 --- a/doc/ci/triggers/README.md +++ b/doc/ci/triggers/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: tutorial --- @@ -20,7 +23,7 @@ The following methods of authentication are supported: - [Trigger token](#trigger-token) - [CI job token](#ci-job-token) -If using the `$CI_PIPELINE_SOURCE` [predefined environment variable](../variables/predefined_variables.md#variables-reference) +If using the `$CI_PIPELINE_SOURCE` [predefined environment variable](../variables/predefined_variables.md) to limit which jobs run in a pipeline, the value could be either `pipeline` or `trigger`, depending on which trigger method is used. @@ -49,7 +52,7 @@ with the [GitLab Container Registry](../../user/packages/container_registry/inde #### When used with multi-project pipelines > - Use of `CI_JOB_TOKEN` for multi-project pipelines was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2017) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.3. -> - Use of `CI_JOB_TOKEN` for multi-project pipelines was [made available](https://gitlab.com/gitlab-org/gitlab/issues/31573) in all tiers in GitLab 12.4. +> - Use of `CI_JOB_TOKEN` for multi-project pipelines was [made available](https://gitlab.com/gitlab-org/gitlab/-/issues/31573) in all tiers in GitLab 12.4. This way of triggering can only be used when invoked inside `.gitlab-ci.yml`, and it creates a dependent pipeline relation visible on the diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 3e31a2169e2..541e739082f 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -17,6 +20,13 @@ that can be reused in different scripts. Variables are useful for customizing your jobs in GitLab CI/CD. When you use variables, you don't have to hard-code values. +> For more information about advanced use of GitLab CI/CD: +> +> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Get to productivity faster with these [7 advanced GitLab CI workflow hacks](https://about.gitlab.com/webcast/7cicd-hacks/) +> shared by GitLab engineers. +> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Learn how the Cloud Native Computing Foundation (CNCF) [eliminates the complexity](https://about.gitlab.com/customers/cncf/) +> of managing projects across many cloud providers with GitLab CI/CD. + ## Predefined environment variables GitLab CI/CD has a [default set of predefined variables](predefined_variables.md) @@ -106,7 +116,7 @@ From within the UI, you can add or update custom environment variables: 1. Go to your project's **Settings > CI/CD** and expand the **Variables** section. 1. Click the **Add Variable** button. In the **Add variable** modal, fill in the details: - - **Key**: Must be one line, with no spaces, using only letters, numbers, `-` or `_`. + - **Key**: Must be one line, with no spaces, using only letters, numbers, or `_`. - **Value**: No limitations. - **Type**: `File` or `Variable`. - **Environment scope**: `All`, or specific environments. @@ -133,7 +143,7 @@ The output will be: ### Custom environment variables of type Variable -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/46806) in GitLab 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/46806) in GitLab 11.11. For variables with the type **Variable**, the Runner creates an environment variable that uses the key for the name and the value for the value. @@ -143,13 +153,13 @@ which may be further validated. They appear when you add or update a variable in ### Custom environment variables of type File -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/46806) in GitLab 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/46806) in GitLab 11.11. For variables with the type **File**, the Runner creates an environment variable that uses the key for the name. For the value, the Runner writes the variable value to a temporary file and uses this path. You can use tools like [the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) -and [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable) +and [`kubectl`](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable) to customize your configuration by using **File** type variables. In the past, a common pattern was to read the value of a CI variable, save it in a file, and then @@ -175,7 +185,7 @@ kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$KU ### Mask a custom variable -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/13784) in GitLab 11.10 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/13784) in GitLab 11.10 Variables can be masked so that the value of the variable will be hidden in job logs. @@ -195,7 +205,7 @@ The value of the variable must: - Be at least 8 characters long. - Not be a predefined or custom environment variable. - Consist only of characters from the Base64 alphabet (RFC4648). - [In GitLab 12.2](https://gitlab.com/gitlab-org/gitlab-foss/issues/63043) + [In GitLab 12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/63043) and newer, `@` and `:` are also valid values. You can't mask variables that don't meet these requirements. @@ -242,13 +252,15 @@ In most cases `bash` or `sh` is used to execute the job script. To access environment variables, use the syntax for your Runner's [shell](https://docs.gitlab.com/runner/executors/). -| Shell | Usage | -|----------------------|-----------------| -| bash/sh | `$variable` | -| windows batch | `%variable%` | -| PowerShell | `$env:variable` | +| Shell | Usage | +|----------------------|------------------------------------------| +| bash/sh | `$variable` | +| PowerShell | `$env:variable` (primary) or `$variable` | +| Windows Batch | `%variable%` | + +### Bash -To access environment variables in bash, prefix the variable name with (`$`): +To access environment variables in **bash**, prefix the variable name with (`$`): ```yaml job_name: @@ -256,32 +268,54 @@ job_name: - echo $CI_JOB_ID ``` -To access environment variables in **Windows Batch**, surround the variable -with (`%`): +### PowerShell + +To access environment variables in a **Windows PowerShell** environment, prefix +the variable name with (`$env:`). For environment variables set by GitLab CI, including those set by [`variables`](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/ci/yaml/README.md#variables) +parameter, they can also be accessed by prefixing the variable name with (`$`) +as of [GitLab Runner 1.0.0](https://gitlab.com/gitlab-org/gitlab-runner/-/commit/abc44bb158008cd3a49c0d8173717c38dadb29ae#47afd7e8f12afdb8f0246262488f24e6dd071a22). +System set environment variables however must be accessed using (`$env:`). ```yaml job_name: script: - - echo %CI_JOB_ID% + - echo $env:CI_JOB_ID + - echo $CI_JOB_ID + - echo $env:PATH ``` -To access environment variables in a **Windows PowerShell** environment, prefix -the variable name with (`$env:`): +In [some cases](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4115#note_157692820) +environment variables may need to be surrounded by quotes to expand properly: ```yaml job_name: script: - - echo $env:CI_JOB_ID + - D:\\qislsf\\apache-ant-1.10.5\\bin\\ant.bat "-DsosposDailyUsr=$env:SOSPOS_DAILY_USR" portal_test ``` -You can also list all environment variables with the `export` command, -but be aware that this will also expose the values of all the variables +### Windows Batch + +To access environment variables in **Windows Batch**, surround the variable +with (`%`): + +```yaml +job_name: + script: + - echo %CI_JOB_ID% +``` + +### List all environment variables + +You can also list all environment variables with the `export` command in Bash +or `dir env:` command in PowerShell. +Be aware that this will also expose the values of all the variables you set, in the job log: ```yaml job_name: script: - export + # - 'dir env:' # use this for PowerShell ``` Example values: @@ -377,9 +411,8 @@ script: You can define per-project or per-group variables that are set in the pipeline environment. Group-level variables are stored out of -the repository (not in `.gitlab-ci.yml`) and are securely passed to GitLab Runner -making them available during a pipeline run. It's the **recommended method** to -use for storing things like passwords, SSH keys, and credentials. +the repository (not in `.gitlab-ci.yml`) and are securely passed to GitLab Runner, +which makes them available during a pipeline run. For Premium users who do **not** use an external key store or who use GitLab's [integration with HashiCorp Vault](../examples/authenticating-with-hashicorp-vault/index.md), we recommend using group environment variables to store secrets like passwords, SSH keys, and credentials. Group-level variables can be added by: @@ -394,10 +427,55 @@ Once you set them, they will be available for all subsequent pipelines. Any grou  -### Inherit environment variables +## Instance-level CI/CD environment variables + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14108) in GitLab 13.0. + +Instance variables are useful for no longer needing to manually enter the same credentials repeatedly for all your projects. Instance-level variables are available to all projects and groups on the instance. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22638) in GitLab 13.0. -> - It's deployed behind a feature flag (`ci_dependency_variables`), disabled by default. +NOTE: **Note:** +The maximum number of instance-level variables is [planned to be 25](https://gitlab.com/gitlab-org/gitlab/-/issues/216097). + +You can define instance-level variables via the UI or [API](../../api/instance_level_ci_variables.md). + +To add an instance-level variable: + +1. Navigate to your admin area's **Settings > CI/CD** and expand the **Variables** section. +1. Click the **Add variable** button, and fill in the details: + + - **Key**: Must be one line, using only letters, numbers, or `_` (underscore), with no spaces. + - **Value**: 700 characters allowed. + - **Type**: `File` or `Variable`. + - **Protect variable** (Optional): If selected, the variable will only be available in pipelines that run on protected branches or tags. + - **Mask variable** (Optional): If selected, the variable's **Value** will not be shown in job logs. The variable will not be saved if the value does not meet the [masking requirements](#masked-variable-requirements). + +After a variable is created, you can update any of the details by clicking the **{pencil}** **Edit** button. + +### Enable or disable UI interface for instance-level CI/CD variables + +The UI interface for Instance-level CI/CD variables is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) can opt to disable it for your instance. + +NOTE: **Note:** +This feature will not work if the [instance-level CI/CD variables API feature flag is disabled](../../api/instance_level_ci_variables.md#enable-or-disable-instance-level-cicd-variables-core-only). + +To disable it: + +```ruby +Feature.disable(:instance_variables_ui) +``` + +To enable it: + +```ruby +Feature.enable(:instance_variables_ui) +``` + +## Inherit environment variables + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22638) in GitLab 13.0 behind a disabled [feature flag](../../administration/feature_flags.md): `ci_dependency_variables`. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/217834) in GitLab 13.1. You can inherit environment variables from dependent jobs. @@ -442,25 +520,6 @@ deploy: artifacts: true ``` -### Enable inherited environment variables **(CORE ONLY)** - -The Inherited Environment Variables feature is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) -can enable it for your instance. - -To enable it: - -```ruby -Feature.enable(:ci_dependency_variables) -``` - -To disable it: - -```ruby -Feature.disable(:ci_dependency_variables) -``` - ## Priority of environment variables Variables of different types can take precedence over other @@ -468,7 +527,8 @@ variables, depending on where they are defined. The order of precedence for variables is (from highest to lowest): -1. [Trigger variables](../triggers/README.md#making-use-of-trigger-variables) or [scheduled pipeline variables](../pipelines/schedules.md#using-variables). +1. [Trigger variables](../triggers/README.md#making-use-of-trigger-variables), [scheduled pipeline variables](../pipelines/schedules.md#using-variables), + and [manual pipeline run variables](#override-a-variable-by-manually-running-a-pipeline). 1. Project-level [variables](#custom-environment-variables) or [protected variables](#protect-a-custom-variable). 1. Group-level [variables](#group-level-environment-variables) or [protected variables](#protect-a-custom-variable). 1. [Inherited environment variables](#inherit-environment-variables). @@ -487,8 +547,10 @@ variables take precedence over those defined in `.gitlab-ci.yml`. ## Unsupported variables -There are cases where some variables cannot be used in the context of a -`.gitlab-ci.yml` definition (for example under `script`). Read more about which variables are [not supported](where_variables_can_be_used.md). +Variable names are limited by the underlying shell used to execute scripts (see [available shells](https://docs.gitlab.com/runner/shells/index.html). +Each shell has its own unique set of reserved variable names. +You will also want to keep in mind the [scope of environment variables](where_variables_can_be_used.md) to ensure a variable is defined in the scope +in which you wish to use it. ## Where variables can be used @@ -518,7 +580,7 @@ An example integration that defines deployment variables is the ### Auto DevOps environment variables -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/49056) in GitLab 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49056) in GitLab 11.7. You can configure [Auto DevOps](../../topics/autodevops/index.md) to pass CI variables to the running application by prefixing the key of the @@ -530,12 +592,12 @@ then be available as environment variables on the running application container. CAUTION: **Caution:** -Variables with multiline values are not currently supported due to +Variables with multi-line values are not currently supported due to limitations with the current Auto DevOps scripting environment. ### Override a variable by manually running a pipeline -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/44059) in GitLab 10.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/44059) in GitLab 10.8. You can override the value of a current variable by [running a pipeline manually](../pipelines/index.md#run-a-pipeline-manually). @@ -555,8 +617,8 @@ value for this specific pipeline. ## Environment variables expressions -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/37397) in GitLab 10.7 for [the `only` and `except` CI keywords](../yaml/README.md#onlyexcept-advanced) -> - [Expanded](https://gitlab.com/gitlab-org/gitlab/issues/27863) in GitLab 12.3 with [the `rules` keyword](../yaml/README.md#rules) +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/37397) in GitLab 10.7 for [the `only` and `except` CI keywords](../yaml/README.md#onlyexcept-advanced) +> - [Expanded](https://gitlab.com/gitlab-org/gitlab/-/issues/27863) in GitLab 12.3 with [the `rules` keyword](../yaml/README.md#rules) Use variable expressions to limit which jobs are created within a pipeline after changes are pushed to GitLab. @@ -612,8 +674,8 @@ Below you can find supported syntax reference: It sometimes happens that you want to check whether a variable is defined or not. To do that, you can compare a variable to `null` keyword, like - `$VARIABLE == null`. This expression is going to evaluate to truth if - variable is not defined when `==` is used, or to falsey if `!=` is used. + `$VARIABLE == null`. This expression evaluates to true if + variable is not defined when `==` is used, or to false if `!=` is used. 1. Checking for an empty variable @@ -644,7 +706,7 @@ Below you can find supported syntax reference: which means that it is defined and non-empty, you can simply use variable name as an expression, like `$STAGING`. If `$STAGING` variable is defined, and is non empty, expression will evaluate to truth. - `$STAGING` value needs to a string, with length higher than zero. + `$STAGING` value needs to be a string, with length higher than zero. Variable that contains only whitespace characters is not an empty variable. 1. Pattern matching (introduced in GitLab 11.0) @@ -652,7 +714,7 @@ Below you can find supported syntax reference: Examples: - `=~`: True if pattern is matched. Ex: `$VARIABLE =~ /^content.*/` - - `!~`: True if pattern is not matched. Ex: `$VARIABLE_1 !~ /^content.*/` ([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/61900) in GitLab 11.11) + - `!~`: True if pattern is not matched. Ex: `$VARIABLE_1 !~ /^content.*/` ([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61900) in GitLab 11.11) Variable pattern matching with regular expressions uses the [RE2 regular expression syntax](https://github.com/google/re2/wiki/Syntax). @@ -694,7 +756,7 @@ deploy_staging: ``` NOTE: **Note:** -The available regular expression syntax is limited. See [related issue](https://gitlab.com/gitlab-org/gitlab/issues/35438) +The available regular expression syntax is limited. See [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/35438) for more details. If needed, you can use a test pipeline to determine whether a regular expression will diff --git a/doc/ci/variables/deprecated_variables.md b/doc/ci/variables/deprecated_variables.md index 543da481938..94ec8439605 100644 --- a/doc/ci/variables/deprecated_variables.md +++ b/doc/ci/variables/deprecated_variables.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/ci/variables/img/ci_job_stage_output_example.png b/doc/ci/variables/img/ci_job_stage_output_example.png Binary files differindex e333da57121..2344b19d9b3 100644 --- a/doc/ci/variables/img/ci_job_stage_output_example.png +++ b/doc/ci/variables/img/ci_job_stage_output_example.png diff --git a/doc/ci/variables/img/inherited_group_variables_v12_5.png b/doc/ci/variables/img/inherited_group_variables_v12_5.png Binary files differindex a13ba711083..cce024cfab8 100644 --- a/doc/ci/variables/img/inherited_group_variables_v12_5.png +++ b/doc/ci/variables/img/inherited_group_variables_v12_5.png diff --git a/doc/ci/variables/img/override_value_via_manual_pipeline_output.png b/doc/ci/variables/img/override_value_via_manual_pipeline_output.png Binary files differindex 8a13bb3849e..2d4c4d24520 100644 --- a/doc/ci/variables/img/override_value_via_manual_pipeline_output.png +++ b/doc/ci/variables/img/override_value_via_manual_pipeline_output.png diff --git a/doc/ci/variables/img/override_variable_manual_pipeline.png b/doc/ci/variables/img/override_variable_manual_pipeline.png Binary files differindex de768105aec..2b242466297 100644 --- a/doc/ci/variables/img/override_variable_manual_pipeline.png +++ b/doc/ci/variables/img/override_variable_manual_pipeline.png diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index d4d3a13bb2a..8463bbeb582 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -7,8 +10,6 @@ type: reference For an introduction on this subject, read through the [getting started with environment variables](README.md) document. -## Overview - Some of the predefined environment variables are available only if a minimum version of [GitLab Runner](https://docs.gitlab.com/runner/) is used. Consult the table below to find the version of Runner required. @@ -19,7 +20,8 @@ Starting with GitLab 9.0, we have deprecated some variables. Read the strongly advised to use the new variables as we will remove the old ones in future GitLab releases.** -## Variables reference +You can add a command to your `.gitlab-ci.yml` file to +[output the values of all variables available for a job](README.md#list-all-environment-variables). | Variable | GitLab | Runner | Description | |-----------------------------------------------|--------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -42,7 +44,7 @@ future GitLab releases.** | `CI_COMMIT_TITLE` | 10.8 | all | The title of the commit - the full first line of the message | | `CI_CONCURRENT_ID` | all | 11.10 | Unique ID of build execution within a single executor. | | `CI_CONCURRENT_PROJECT_ID` | all | 11.10 | Unique ID of build execution within a single executor and project. | -| `CI_CONFIG_PATH` | 9.4 | 0.5 | The path to CI config file. Defaults to `.gitlab-ci.yml` | +| `CI_CONFIG_PATH` | 9.4 | 0.5 | The path to CI configuration file. Defaults to `.gitlab-ci.yml` | | `CI_DEBUG_TRACE` | all | 1.7 | Whether [debug logging (tracing)](README.md#debug-logging) is enabled | | `CI_DEFAULT_BRANCH` | 12.4 | all | The name of the default branch for the project. | | `CI_DEPLOY_PASSWORD` | 10.8 | all | Authentication password of the [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token), only present if the Project has one related. | @@ -56,18 +58,17 @@ future GitLab releases.** | `CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_SHA` | 12.3 | all | The HEAD SHA of the source branch of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | | `CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME` | 12.3 | all | The target branch name of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | | `CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA` | 12.3 | all | The HEAD SHA of the target branch of the pull request if [the pipelines are for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). Available only if `only: [external_pull_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the pull request is open. | +| `CI_HAS_OPEN_REQUIREMENTS` | 13.1 | all | Included with the value `true` only if the pipeline's project has any open [requirements](../../user/project/requirements/index.md). Not included if there are no open requirements for the pipeline's project. | | `CI_JOB_ID` | 9.0 | all | The unique ID of the current job that GitLab CI/CD uses internally | | `CI_JOB_IMAGE` | 12.9 | 12.9 | The name of the image running the CI job | | `CI_JOB_MANUAL` | 8.12 | all | The flag to indicate that job was manually started | | `CI_JOB_NAME` | 9.0 | 0.5 | The name of the job as defined in `.gitlab-ci.yml` | | `CI_JOB_STAGE` | 9.0 | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` | -| `CI_JOB_TOKEN` | 9.0 | 1.2 | Token used for authenticating with the [GitLab Container Registry](../../user/packages/container_registry/index.md) and downloading [dependent repositories](../../user/project/new_ci_build_permissions_model.md#dependent-repositories) | +| `CI_JOB_TOKEN` | 9.0 | 1.2 | Token used for authenticating with the [GitLab Container Registry](../../user/packages/container_registry/index.md), downloading [dependent repositories](../../user/project/new_ci_build_permissions_model.md#dependent-repositories), and accessing [GitLab-managed Terraform state](../../user/infrastructure/index.md#gitlab-managed-terraform-state). | | `CI_JOB_JWT` | 12.10 | all | RS256 JSON web token that can be used for authenticating with third party systems that support JWT authentication, for example [HashiCorp's Vault](../examples/authenticating-with-hashicorp-vault). | | `CI_JOB_URL` | 11.1 | 0.5 | Job details URL | -| `CI_KUBERNETES_ACTIVE` | 13.0 | all | Included with the value `true` only if the pipeline has a Kubernetes cluster available for deployments. Not included if no cluster is availble. Can be used as an alternative to [`only:kubernetes`/`except:kubernetes`](../yaml/README.md#onlykubernetesexceptkubernetes) with [`rules:if`](../yaml/README.md#rulesif) | +| `CI_KUBERNETES_ACTIVE` | 13.0 | all | Included with the value `true` only if the pipeline has a Kubernetes cluster available for deployments. Not included if no cluster is available. Can be used as an alternative to [`only:kubernetes`/`except:kubernetes`](../yaml/README.md#onlykubernetesexceptkubernetes) with [`rules:if`](../yaml/README.md#rulesif) | | `CI_MERGE_REQUEST_ASSIGNEES` | 11.9 | all | Comma-separated list of username(s) of assignee(s) for the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | -| `CI_MERGE_REQUEST_CHANGED_PAGE_PATHS` | 12.9 | all | Comma-separated list of paths of changed pages in a deployed [Review App](../review_apps/index.md) for a [Merge Request](../merge_request_pipelines/index.md). A [Route Map](../review_apps/index.md#route-maps) must be configured. | -| `CI_MERGE_REQUEST_CHANGED_PAGE_URLS` | 12.9 | all | Comma-separated list of URLs of changed pages in a deployed [Review App](../review_apps/index.md) for a [Merge Request](../merge_request_pipelines/index.md). A [Route Map](../review_apps/index.md#route-maps) must be configured. | | `CI_MERGE_REQUEST_ID` | 11.6 | all | The ID of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | | `CI_MERGE_REQUEST_IID` | 11.6 | all | The IID of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | | `CI_MERGE_REQUEST_LABELS` | 11.9 | all | Comma-separated label names of the merge request if [the pipelines are for merge requests](../merge_request_pipelines/index.md). Available only if `only: [merge_requests]` or [`rules`](../yaml/README.md#rules) syntax is used and the merge request is created. | @@ -97,7 +98,7 @@ future GitLab releases.** | `CI_PROJECT_DIR` | all | all | The full path where the repository is cloned and where the job is run. If the GitLab Runner `builds_dir` parameter is set, this variable is set relative to the value of `builds_dir`. For more information, see [Advanced configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) for GitLab Runner. | | `CI_PROJECT_ID` | all | all | The unique ID of the current project that GitLab CI/CD uses internally | | `CI_PROJECT_NAME` | 8.10 | 0.5 | The name of the directory for the project that is currently being built. For example, if the project URL is `gitlab.example.com/group-name/project-1`, the `CI_PROJECT_NAME` would be `project-1`. | -| `CI_PROJECT_NAMESPACE` | 8.10 | 0.5 | The project namespace (username or groupname) that is currently being built | +| `CI_PROJECT_NAMESPACE` | 8.10 | 0.5 | The project namespace (username or group name) that is currently being built | | `CI_PROJECT_PATH` | 8.10 | 0.5 | The namespace with project name | | `CI_PROJECT_PATH_SLUG` | 9.3 | all | `$CI_PROJECT_PATH` lowercased and with everything except `0-9` and `a-z` replaced with `-`. Use in URLs and domain names. | | `CI_PROJECT_REPOSITORY_LANGUAGES` | 12.3 | all | Comma-separated, lowercased list of the languages used in the repository (e.g. `ruby,javascript,html,css`) | diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index 0c40ac960e5..2f26fddc808 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -72,10 +75,10 @@ ordering of variables definitions. ### Execution shell environment This is an expansion that takes place during the `script` execution. -How it works depends on the used shell (bash/sh/cmd/PowerShell). For example, if the job's +How it works depends on the used shell (`bash`, `sh`, `cmd`, PowerShell). For example, if the job's `script` contains a line `echo $MY_VARIABLE-${MY_VARIABLE_2}`, it should be properly handled by bash/sh (leaving empty strings or some values depending whether the variables were -defined or not), but will not work with Windows' cmd/PowerShell, since these shells +defined or not), but will not work with Windows' `cmd` or PowerShell, since these shells are using a different variables syntax. Supported: diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index c40580cbfb7..7ed5a8fec01 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -22,6 +25,14 @@ We have complete examples of configuring pipelines: - For a collection of examples, see [GitLab CI/CD Examples](../examples/README.md). - To see a large `.gitlab-ci.yml` file used in an enterprise, see the [`.gitlab-ci.yml` file for `gitlab`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml). +> For some additional information about GitLab CI/CD: +> +> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch the [CI/CD Ease of configuration](https://www.youtube.com/embed/opdLqwz6tcE) video. +> - Watch the [Making the case for CI/CD in your organization](https://about.gitlab.com/compare/github-actions-alternative/) +> webcast to learn the benefits of CI/CD and how to measure the results of CI/CD automation. +> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Learn how [Verizon reduced rebuilds](https://about.gitlab.com/blog/2019/02/14/verizon-customer-story/) +> from 30 days to under 8 hours with GitLab. + NOTE: **Note:** If you have a [mirrored repository where GitLab pulls from](../../user/project/repository/repository_mirroring.md#pulling-from-a-remote-repository-starter), you may need to enable pipeline triggering in your project's @@ -93,8 +104,8 @@ The following table lists available parameters for jobs: | Keyword | Description | |:---------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [`script`](#script) | Shell script which is executed by Runner. | -| [`image`](#image) | Use docker images. Also available: `image:name` and `image:entrypoint`. | -| [`services`](#services) | Use docker services images. Also available: `services:name`, `services:alias`, `services:entrypoint`, and `services:command`. | +| [`image`](#image) | Use Docker images. Also available: `image:name` and `image:entrypoint`. | +| [`services`](#services) | Use Docker services images. Also available: `services:name`, `services:alias`, `services:entrypoint`, and `services:command`. | | [`before_script`](#before_script-and-after_script) | Override a set of commands that are executed before job. | | [`after_script`](#before_script-and-after_script) | Override a set of commands that are executed after job. | | [`stage`](#stage) | Defines a job stage (default: `test`). | @@ -106,7 +117,7 @@ The following table lists available parameters for jobs: | [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | | [`environment`](#environment) | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. | | [`cache`](#cache) | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, and `cache:policy`. | -| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:codequality`, `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_scanning`, `artifacts:reports:license_management` (removed in GitLab 13.0),`artifacts:reports:performance` and `artifacts:reports:metrics`. | +| [`artifacts`](#artifacts) | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:exclude`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:codequality`, `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_scanning`, `artifacts:reports:license_management` (removed in GitLab 13.0),`artifacts:reports:performance` and `artifacts:reports:metrics`. | | [`dependencies`](#dependencies) | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from. | | [`coverage`](#coverage) | Code coverage settings for a given job. | | [`retry`](#retry) | When and how many times a job can be auto-retried in case of a failure. | @@ -163,7 +174,7 @@ rspec 2.6: #### `inherit` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/207484) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207484) in GitLab 12.9. You can disable inheritance of globally defined defaults and variables with the `inherit:` parameter. @@ -279,7 +290,7 @@ There are also two edge cases worth mentioning: ### `workflow:rules` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29654) in GitLab 12.5 +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29654) in GitLab 12.5 The top-level `workflow:` key applies to the entirety of a pipeline, and will determine whether or not a pipeline is created. It currently accepts a single @@ -310,7 +321,6 @@ include: - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' ``` -The [`MergeRequest-Pipelines` include](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml) sets your pipelines to run for the default branch (usually `master`), tags, and The [`MergeRequest-Pipelines` template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml) makes your pipelines run for the default branch (usually `master`), tags, and all types of merge request pipelines. Use this template if you use any of the @@ -418,7 +428,7 @@ include: '.gitlab-ci-production.yml' #### `include:file` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53903) in GitLab 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53903) in GitLab 11.7. To include files from another private project under the same GitLab instance, use `include:file`. This file is referenced using full paths relative to the @@ -468,7 +478,7 @@ or public project, or template, is allowed. #### `include:template` -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53445) in GitLab 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53445) in GitLab 11.7. `include:template` can be used to include `.gitlab-ci.yml` templates that are [shipped with GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates). @@ -494,13 +504,13 @@ so it's possible to use project, remote or template includes. #### Nested includes -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/56836) in GitLab 11.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/56836) in GitLab 11.9. Nested includes allow you to compose a set of includes. A total of 100 includes is allowed, but duplicate includes are considered a configuration error. -Since [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/28212), the time limit +Since [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/28212), the time limit for resolving all files is 30 seconds. #### Additional `includes` examples @@ -522,13 +532,13 @@ For: #### `image:name` -An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). +An [extended Docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `image`](../docker/using_docker_images.md#available-settings-for-image). #### `image:entrypoint` -An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). +An [extended Docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `image`](../docker/using_docker_images.md#available-settings-for-image). @@ -544,25 +554,25 @@ For: ##### `services:name` -An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). +An [extended Docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). ##### `services:alias` -An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). +An [extended Docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). ##### `services:entrypoint` -An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). +An [extended Docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). ##### `services:command` -An [extended docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). +An [extended Docker configuration option](../docker/using_docker_images.md#extended-docker-configuration-options). For more information, see [Available settings for `services`](../docker/using_docker_images.md#available-settings-for-services). @@ -628,7 +638,7 @@ Scripts specified in `after_script` are executed in a new shell, separate from a - Changes outside of the working tree (depending on the Runner executor), like software installed by a `before_script` or `script` script. - Have a separate timeout, which is hard coded to 5 minutes. See - [related issue](https://gitlab.com/gitlab-org/gitlab-runner/issues/2716) for details. + [related issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716) for details. - Don't affect the job's exit code. If the `script` section succeeds and the `after_script` times out or fails, the job will exit with code `0` (`Job Succeeded`). @@ -651,6 +661,44 @@ job: [YAML anchors for `before_script` and `after_script`](#yaml-anchors-for-before_script-and-after_script) are available. +#### Coloring script output + +Script output can be colored using [ANSI escape codes](https://en.wikipedia.org/wiki/ANSI_escape_code#Colors), +or by running commands or programs that output ANSI escape codes. + +For example, using [Bash with color codes](https://misc.flogisoft.com/bash/tip_colors_and_formatting): + +```yaml +job: + script: + - echo -e "\e[31mThis text is red,\e[0m but this text isn't\e[31m however this text is red again." +``` + +You can define the color codes in Shell variables, or even [custom environment variables](../variables/README.md#custom-environment-variables), +which makes the commands easier to read and reusable. + +For example, using the same example as above and variables defined in a `before_script`: + +```yaml +job: + before_script: + - TXT_RED="\e[31m" && TXT_CLEAR="\e[0m" + script: + - echo -e "${TXT_RED}This text is red,${TXT_CLEAR} but this part isn't${TXT_RED} however this part is again." + - echo "This text is not colored" +``` + +Or with [PowerShell color codes](https://superuser.com/a/1259916): + +```yaml +job: + before_script: + - $esc="$([char]27)"; $TXT_RED="$esc[31m"; $TXT_CLEAR="$esc[0m" + script: + - Write-Host $TXT_RED"This text is red,"$TXT_CLEAR" but this text isn't"$TXT_RED" however this text is red again." + - Write-Host "This text is not colored" +``` + ### `stage` `stage` is defined per-job and relies on [`stages`](#stages) which is defined @@ -701,7 +749,7 @@ Jobs will run on your own Runners in parallel only if: #### `.pre` and `.post` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31441) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31441) in GitLab 12.4. The following stages are available to every pipeline: @@ -910,7 +958,7 @@ can use one of the [`workflow: rules` templates](#workflowrules-templates) to ge This will ensure that the behavior is more stable as you start adding additional `rules` blocks, and will avoid issues like creating a duplicate, merge request (detached) pipeline. -We don't recomment mixing `only/except` jobs with `rules` jobs in the same pipeline. +We don't recommend mixing `only/except` jobs with `rules` jobs in the same pipeline. It may not cause YAML errors, but debugging the exact execution behavior can be complex due to the different default behaviors of `only/except` and `rules`. @@ -966,10 +1014,14 @@ job: - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' # If neither of the first two match but the simple presence does, we set to "on_success" by default ``` -If none of the provided rules match, the job will be set to `when:never`, and -not included in the pipeline. If `rules:when` is not included in the configuration -at all, the behavior defaults to `job:when`, which continues to default to -`on_success`. +Some details regarding the logic that determines the `when` for the job: + +- If none of the provided rules match, the job is set to `when: never`, and is + not included in the pipeline. +- A rule without any conditional clause, such as a `when` or `allow_failure` + rule without `if` or `changes`, always matches, and is always used if reached. +- If a rule matches and has no `when` defined, the rule will use the `when` + defined for the job, which defaults to `on_success` if not defined. #### `rules:changes` @@ -1032,12 +1084,16 @@ checks. After the 10000th check, rules with patterned globs will always match. #### `rules:allow_failure` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30235) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30235) in GitLab 12.8. You can use [`allow_failure: true`](#allow_failure) within `rules:` to allow a job to fail, or a manual job to wait for action, without stopping the pipeline itself. All jobs using `rules:` default to `allow_failure: false` if `allow_failure:` is not defined. +The rule-level `rules:allow_failure` option overrides the job-level +[`allow_failure`](#allow_failure) option, and is only applied when the job is +triggered by the particular rule. + ```yaml job: script: "echo Hello, Rules!" @@ -1111,7 +1167,7 @@ docker build: Additional job configuration may be added to rules in the future. If something useful is not available, please -[open an issue](https://gitlab.com/gitlab-org/gitlab/issues). +[open an issue](https://gitlab.com/gitlab-org/gitlab/-/issues). ### `only`/`except` (basic) @@ -1402,7 +1458,7 @@ Learn more about [variables expressions](../variables/README.md#environment-vari #### `only:changes`/`except:changes` -> `changes` policy [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/19232) in GitLab 11.4. +> `changes` policy [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/19232) in GitLab 11.4. Using the `changes` keyword with `only` or `except` makes it possible to define if a job should be created based on files modified by a Git push event. @@ -1414,7 +1470,7 @@ This means the `only:changes` policy is useful for pipelines where: - `$CI_PIPELINE_SOURCE == 'external_pull_request_event'` If there is no Git push event, such as for pipelines with -[sources other than the three above](../variables/predefined_variables.md#variables-reference), +[sources other than the three above](../variables/predefined_variables.md), `changes` can't determine if a given file is new or old, and will always return true. @@ -1548,9 +1604,9 @@ runs. ### `needs` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47063) in GitLab 12.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) in GitLab 12.2. > - In GitLab 12.3, maximum number of jobs in `needs` array raised from five to 50. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30631) in GitLab 12.8, `needs: []` lets jobs start immediately. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30631) in GitLab 12.8, `needs: []` lets jobs start immediately. The `needs:` keyword enables executing jobs out-of-order, allowing you to implement a [directed acyclic graph](../directed_acyclic_graph/index.md) in your `.gitlab-ci.yml`. @@ -1612,7 +1668,7 @@ This example creates four paths of execution: pipeline will be created with YAML error. - The maximum number of jobs that a single job can need in the `needs:` array is limited: - For GitLab.com, the limit is ten. For more information, see our - [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/7541). + [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7541). - For self-managed instances, the limit is: - 10, if the `ci_dag_limit_needs` feature flag is enabled (default). - 50, if the `ci_dag_limit_needs` feature flag is disabled. @@ -1620,7 +1676,7 @@ This example creates four paths of execution: the current job will depend on all parallel jobs created. - `needs:` is similar to `dependencies:` in that it needs to use jobs from prior stages, meaning it's impossible to create circular dependencies. Depending on jobs in the - current stage is not possible either, but support [is planned](https://gitlab.com/gitlab-org/gitlab/issues/30632). + current stage is not possible either, but support [is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/30632). - Related to the above, stages must be explicitly defined for all jobs that have the keyword `needs:` or are referred to by one. @@ -1643,7 +1699,7 @@ Feature::enable(:ci_dag_limit_needs) #### Artifact downloads with `needs` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14311) in GitLab v12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14311) in GitLab v12.6. When using `needs`, artifact downloads are controlled with `artifacts: true` or `artifacts: false`. The `dependencies` keyword should not be used with `needs`, as this is deprecated since GitLab 12.6. @@ -1686,7 +1742,7 @@ rspec: #### Cross project artifact downloads with `needs` **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14311) in GitLab v12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14311) in GitLab v12.7. `needs` can be used to download artifacts from up to five jobs in pipelines on [other refs in the same project](#artifact-downloads-between-pipelines-in-the-same-project), @@ -2142,9 +2198,15 @@ The `stop_review_app` job is **required** to have the following keywords defined - `stage` should be the same as the `review_app` in order for the environment to stop automatically when the branch is deleted +Additionally, both jobs should have matching [`rules`](../yaml/README.md#onlyexcept-basic) +or [`only/except`](../yaml/README.md#onlyexcept-basic) configuration. In the example +above, if the configuration is not identical, the `stop_review_app` job might not be +included in all pipelines that include the `review_app` job, and it will not be +possible to trigger the `action: stop` to stop the environment automatically. + #### `environment:auto_stop_in` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/20956) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20956) in GitLab 12.8. The `auto_stop_in` keyword is for specifying life period of the environment, that when expired, GitLab automatically stops them. @@ -2167,7 +2229,7 @@ For more information, see #### `environment:kubernetes` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27630) in GitLab 12.6. The `kubernetes` block is used to configure deployments to a [Kubernetes cluster](../../user/project/clusters/index.md) that is associated with your project. @@ -2195,7 +2257,7 @@ NOTE: **Note:** Kubernetes configuration is not supported for Kubernetes clusters that are [managed by GitLab](../../user/project/clusters/index.md#gitlab-managed-clusters). To follow progress on support for GitLab-managed clusters, see the -[relevant issue](https://gitlab.com/gitlab-org/gitlab/issues/38054). +[relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). #### Dynamic environments @@ -2336,7 +2398,7 @@ cache: ##### `cache:key:files` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/18986) in GitLab v12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18986) in GitLab v12.5. The `cache:key:files` keyword extends the `cache:key` functionality by making it easier to reuse some caches, and rebuild them less often, which will speed up subsequent pipeline @@ -2366,7 +2428,7 @@ use the new cache, instead of rebuilding the dependencies. ##### `cache:key:prefix` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/18986) in GitLab v12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18986) in GitLab v12.5. The `prefix` parameter adds extra functionality to `key:files` by allowing the key to be composed of the given `prefix` combined with the SHA computed for `cache:key:files`. @@ -2545,9 +2607,36 @@ job: - path/*xyz/* ``` +#### `artifacts:exclude` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15122) in GitLab 13.1 +> - Requires GitLab Runner 13.1 + +`exclude` makes it possible to prevent files from being added to an artifacts +archive. + +Similar to [`artifacts:paths`](#artifactspaths), `exclude` paths are relative +to the project directory. Wildcards can be used that follow the +[glob](https://en.wikipedia.org/wiki/Glob_(programming)) patterns and +[`filepath.Match`](https://golang.org/pkg/path/filepath/#Match). + +For example, to store all files in `binaries/`, but not `*.o` files located in +subdirectories of `binaries/`: + +```yaml +artifacts: + paths: + - binaries/ + exclude: + - binaries/**/*.o +``` + +Files matched by [`artifacts:untracked`](#artifactsuntracked) can be excluded using +`artifacts:exclude` too. + #### `artifacts:expose_as` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/15018) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15018) in GitLab 12.5. The `expose_as` keyword can be used to expose [job artifacts](../pipelines/job_artifacts.md) in the [merge request](../../user/project/merge_requests/index.md) UI. @@ -2691,6 +2780,15 @@ artifacts: - binaries/ ``` +Send all untracked files but [exclude](#artifactsexclude) `*.txt`: + +```yaml +artifacts: + untracked: true + exclude: + - *.txt +``` + #### `artifacts:when` > Introduced in GitLab 8.9 and GitLab Runner v1.3.0. @@ -2748,7 +2846,7 @@ job: ``` NOTE: **Note:** -For artifacts created in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/16267) +For artifacts created in [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/16267) and later, the latest artifact for a ref is always kept, regardless of the expiry time. #### `artifacts:reports` @@ -2948,7 +3046,7 @@ You can specify the number of [retry attempts for certain stages of job executio ### `timeout` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14887) in GitLab 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14887) in GitLab 12.3. `timeout` allows you to configure a timeout for a specific job. For example: @@ -3017,8 +3115,8 @@ job split into three separate jobs. ### `trigger` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/199224) to GitLab Core in 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/199224) to GitLab Core in 12.8. `trigger` allows you to define downstream pipeline trigger. When a job created from `trigger` definition is started by GitLab, a downstream pipeline gets @@ -3086,7 +3184,7 @@ upstream_bridge: #### `trigger` syntax for child pipeline -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/16094) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16094) in GitLab 12.7. To create a [child pipeline](../parent_child_pipelines.md), specify the path to the YAML file containing the CI config of the child pipeline: @@ -3110,7 +3208,7 @@ trigger_job: ##### Trigger child pipeline with generated configuration file -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/35632) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35632) in GitLab 12.9. You can also trigger a child pipeline from a [dynamically generated configuration file](../parent_child_pipelines.md#dynamic-child-pipelines): @@ -3217,7 +3315,7 @@ Once an uninterruptible job is running, the pipeline will never be canceled, reg ### `resource_group` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/15536) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15536) in GitLab 12.7. Sometimes running multiples jobs or pipelines at the same time in an environment can lead to errors during the deployment. @@ -3252,6 +3350,8 @@ NOTE: **Note:** This key can only contain letters, digits, `-`, `_`, `/`, `$`, `{`, `}`, `.`, and spaces. It can't start or end with `/`. +For more information, see [Deployments Safety](../environments/deployment_safety.md). + ### `pages` `pages` is a special job that is used to upload static content to GitLab that @@ -3351,7 +3451,7 @@ variables: `none` also re-uses the local working copy, but skips all Git operations (including GitLab Runner's pre-clone script, if present). It's mostly useful -for jobs that operate exclusively on artifacts (for examples `deploy`). Git repository +for jobs that operate exclusively on artifacts (for example, `deploy`). Git repository data may be present, but it's certain to be out of date, so you should only rely on files brought into the local working copy from cache or artifacts. @@ -3362,7 +3462,7 @@ variables: NOTE: **Note:** `GIT_STRATEGY` is not supported for [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html), -but may be in the future. See the [support Git strategy with Kubernetes executor feature proposal](https://gitlab.com/gitlab-org/gitlab-runner/issues/3847) +but may be in the future. See the [support Git strategy with Kubernetes executor feature proposal](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3847) for updates. ### Git submodule strategy @@ -3459,6 +3559,43 @@ script: - ls -al cache/ ``` +### Git fetch extra flags + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4142) in GitLab Runner 13.1. + +The `GIT_FETCH_EXTRA_FLAGS` variable is used to control the behavior of +`git fetch`. You can set it globally or per-job in the [`variables`](#variables) section. + +`GIT_FETCH_EXTRA_FLAGS` accepts all possible options of the [`git fetch`](https://git-scm.com/docs/git-fetch) command, but please note that `GIT_FETCH_EXTRA_FLAGS` flags will be appended after the default flags that can't be modified. + +The default flags are: + +- [GIT_DEPTH](#shallow-cloning). +- The list of [refspecs](https://git-scm.com/book/en/v2/Git-Internals-The-Refspec). +- A remote called `origin`. + +If `GIT_FETCH_EXTRA_FLAGS` is: + +- Not specified, `git fetch` flags default to `--prune --quiet` along with the default flags. +- Given the value `none`, `git fetch` is executed only with the default flags. + +For example, the default flags are `--prune --quiet`, so you can make `git fetch` more verbose by overriding this with just `--prune`: + +```yaml +variables: + GIT_FETCH_EXTRA_FLAGS: --prune +script: + - ls -al cache/ +``` + +The configurtion above will result in `git fetch` being called this way: + +```shell +git fetch origin $REFSPECS --depth 50 --prune +``` + +Where `$REFSPECS` is a value provided to the Runner internally by GitLab. + ### Job stages attempts > Introduced in GitLab, it requires GitLab Runner v1.9+. @@ -3488,8 +3625,8 @@ You can set them globally or per-job in the [`variables`](#variables) section. > Introduced in GitLab 8.9 as an experimental feature. -CAUTION: **Caution:** -May change in future releases or be removed completely. +NOTE: **Note**: +As of GitLab 12.0, newly created projects will automatically have a [default `git depth` value of `50`](../pipelines/settings.md#git-shallow-clone). You can specify the depth of fetching and cloning using `GIT_DEPTH`. This allows shallow cloning of the repository which can significantly speed up cloning for @@ -3739,7 +3876,7 @@ feature. Anchors are only valid within the file they were defined in. #### YAML anchors for `before_script` and `after_script` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/23005) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23005) in GitLab 12.5. You can use [YAML anchors](#anchors) with `before_script` and `after_script`, which makes it possible to include a predefined list of commands in multiple @@ -3753,7 +3890,7 @@ Example: .something_after: &something_after - echo 'something after' - +- echo 'another thing after' job_name: before_script: @@ -3766,7 +3903,7 @@ job_name: #### YAML anchors for `script` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/23005) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23005) in GitLab 12.5. You can use [YAML anchors](#anchors) with scripts, which makes it possible to include a predefined list of commands in multiple jobs. @@ -3796,6 +3933,7 @@ the use of the `SAMPLE_VARIABLE` variable: # global variables variables: &global-variables SAMPLE_VARIABLE: sample_variable_value + ANOTHER_SAMPLE_VARIABLE: another_sample_variable_value # a job that needs to set the GIT_STRATEGY variable, yet depend on global variables job_no_git_strategy: @@ -3819,7 +3957,7 @@ lines where the job is defined: # - run test ``` -you can instead start its name with a dot (`.`) and it won't be processed by +You can instead start its name with a dot (`.`) and it won't be processed by GitLab CI/CD. In the following example, `.hidden_job` will be ignored: ```yaml @@ -3842,11 +3980,11 @@ if using Git 2.10 or newer. ## Processing Git pushes -GitLab will create at most 4 branch and tags pipelines when -doing pushing multiple changes in single `git push` invocation. +GitLab will create at most 4 branch and tag pipelines when +pushing multiple changes in single `git push` invocation. -This limitation does not affect any of the updated Merge Request pipelines, -all updated Merge Requests will have a pipeline created when using +This limitation does not affect any of the updated Merge Request pipelines. +All updated Merge Requests will have a pipeline created when using [pipelines for merge requests](../merge_request_pipelines/index.md). ## Deprecated parameters diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index a7b626bdd7c..969b54d5be8 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -1,3 +1,10 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + # GitLab CI/CD YAML includes In addition to the [`includes` examples](README.md#include) listed in the diff --git a/doc/development/README.md b/doc/development/README.md index 22cc21e12b9..d330d6d466e 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -63,7 +63,7 @@ Complementary reads: - [Working with Gitaly](gitaly.md) - [Manage feature flags](feature_flags/index.md) - [Licensed feature availability](licensed_feature_availability.md) -- [View sent emails or preview mailers](emails.md) +- [Dealing with email/mailers](emails.md) - [Shell commands](shell_commands.md) in the GitLab codebase - [`Gemfile` guidelines](gemfile.md) - [Pry debugging](pry_debugging.md) @@ -113,50 +113,7 @@ Complementary reads: ## Database guides -### Tooling - -- [Understanding EXPLAIN plans](understanding_explain_plans.md) -- [explain.depesz.com](https://explain.depesz.com/) for visualizing the output - of `EXPLAIN` -- [pgFormatter](http://sqlformat.darold.net/) a PostgreSQL SQL syntax beautifier - -### Migrations - -- [What requires downtime?](what_requires_downtime.md) -- [SQL guidelines](sql.md) for working with SQL queries -- [Migrations style guide](migration_style_guide.md) for creating safe SQL migrations -- [Testing Rails migrations](testing_guide/testing_migrations_guide.md) guide -- [Post deployment migrations](post_deployment_migrations.md) -- [Background migrations](background_migrations.md) -- [Swapping tables](swapping_tables.md) -- [Deleting migrations](deleting_migrations.md) - -### Debugging - -- Tracing the source of an SQL query using query comments with [Marginalia](database_query_comments.md) -- Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs) - -### Best practices - -- [Adding database indexes](adding_database_indexes.md) -- [Foreign keys & associations](foreign_keys.md) -- [Single table inheritance](single_table_inheritance.md) -- [Polymorphic associations](polymorphic_associations.md) -- [Serializing data](serializing_data.md) -- [Hash indexes](hash_indexes.md) -- [Storing SHA1 hashes as binary](sha1_as_binary.md) -- [Iterating tables in batches](iterating_tables_in_batches.md) -- [Insert into tables in batches](insert_into_tables_in_batches.md) -- [Ordering table columns](ordering_table_columns.md) -- [Verifying database capabilities](verifying_database_capabilities.md) -- [Database Debugging and Troubleshooting](database_debugging.md) -- [Query Count Limits](query_count_limits.md) -- [Creating enums](creating_enums.md) - -### Case studies - -- [Database case study: Filtering by label](filtering_by_label.md) -- [Database case study: Namespaces storage statistics](namespaces_storage_statistics.md) +See [database guidelines](database/index.md). ## Integration guides diff --git a/doc/development/adding_database_indexes.md b/doc/development/adding_database_indexes.md index 01b621b6631..7fe047b380b 100644 --- a/doc/development/adding_database_indexes.md +++ b/doc/development/adding_database_indexes.md @@ -20,8 +20,8 @@ existing indexes need to be updated. The more indexes there are the slower this can potentially become. Indexes can also take up quite some disk space depending on the amount of data indexed and the index type. For example, PostgreSQL offers "GIN" indexes which can be used to index certain data types that can not be -indexed by regular btree indexes. These indexes however generally take up more -data and are slower to update compared to btree indexes. +indexed by regular B-tree indexes. These indexes however generally take up more +data and are slower to update compared to B-tree indexes. Because of all this one should not blindly add a new index for every column used to filter data by. Instead one should ask themselves the following questions: diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 6d3c0cf0eac..43e96340d10 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -346,7 +346,7 @@ GitLab's GraphQL API is versionless, which means we maintain backwards compatibility with older versions of the API with every change. Rather than removing a field, we need to _deprecate_ the field instead. In future, GitLab -[may remove deprecated fields](https://gitlab.com/gitlab-org/gitlab/issues/32292). +[may remove deprecated fields](https://gitlab.com/gitlab-org/gitlab/-/issues/32292). Fields are deprecated using the `deprecated` property. The value of the property is a `Hash` of: @@ -470,6 +470,23 @@ field :confidential, GraphQL::BOOLEAN_TYPE, description: 'Indicates the issue is field :closed_at, Types::TimeType, description: 'Timestamp of when the issue was closed' ``` +### `copy_field_description` helper + +Sometimes we want to ensure that two descriptions will always be identical. +For example, to keep a type field description the same as a mutation argument +when they both represent the same property. + +Instead of supplying a description, we can use the `copy_field_description` helper, +passing it the type, and field name to copy the description of. + +Example: + +```ruby +argument :title, GraphQL::STRING_TYPE, + required: false, + description: copy_field_description(Types::MergeRequestType, :title) +``` + ## Authorization Authorizations can be applied to both types and fields using the same @@ -602,6 +619,84 @@ lot of dependent objects. To limit the amount of queries performed, we can use `BatchLoader`. +### Correct use of `Resolver#ready?` + +Resolvers have two public API methods as part of the framework: `#ready?(**args)` and `#resolve(**args)`. +We can use `#ready?` to perform set-up, validation or early-return without invoking `#resolve`. + +Good reasons to use `#ready?` include: + +- validating mutually exclusive arguments (see [validating arguments](#validating-arguments)) +- Returning `Relation.none` if we know before-hand that no results are possible +- Performing setup such as initializing instance variables (although consider lazily initialized methods for this) + +Implementations of [`Resolver#ready?(**args)`](https://graphql-ruby.org/api-doc/1.10.9/GraphQL/Schema/Resolver#ready%3F-instance_method) should +return `(Boolean, early_return_data)` as follows: + +```ruby +def ready?(**args) + [false, 'have this instead'] +end +``` + +For this reason, whenever you call a resolver (mainly in tests - as framework +abstractions Resolvers should not be considered re-usable, finders are to be +preferred), remember to call the `ready?` method and check the boolean flag +before calling `resolve`! An example can be seen in our [`GraphQLHelpers`](https://gitlab.com/gitlab-org/gitlab/-/blob/2d395f32d2efbb713f7bc861f96147a2a67e92f2/spec/support/helpers/graphql_helpers.rb#L20-27). + +### Look-Ahead + +The full query is known in advance during execution, which means we can make use +of [lookahead](https://graphql-ruby.org/queries/lookahead.html) to optimize our +queries, and batch load associations we know we will need. Consider adding +lookahead support in your resolvers to avoid `N+1` performance issues. + +To enable support for common lookahead use-cases (pre-loading associations when +child fields are requested), you can +include [`LooksAhead`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/resolvers/concerns/looks_ahead.rb). For example: + +```ruby +# Assuming a model `MyThing` with attributes `[child_attribute, other_attribute, nested]`, +# where nested has an attribute named `included_attribute`. +class MyThingResolver < BaseResolver + include LooksAhead + + # Rather than defining `resolve(**args)`, we implement: `resolve_with_lookahead(**args)` + def resolve_with_lookahead(**args) + apply_lookahead(MyThingFinder.new(current_user).execute) + end + + # We list things that should always be preloaded: + # For example, if child_attribute is always needed (during authorization + # perhaps), then we can include it here. + def unconditional_includes + [:child_attribute] + end + + # We list things that should be included if a certain field is selected: + def preloads + { + field_one: [:other_attribute], + field_two: [{ nested: [:included_attribute] }] + } + end +end +``` + +The final thing that is needed is that every field that uses this resolver needs +to advertise the need for lookahead: + +```ruby + # in ParentType + field :my_things, MyThingType.connection_type, null: true, + extras: [:lookahead], # Necessary + resolver: MyThingResolver, + description: 'My things' +``` + +For an example of real world use, please +see [`ResolvesMergeRequests`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/resolvers/concerns/resolves_merge_requests.rb). + ## Mutations Mutations are used to change any stored values, or to trigger @@ -613,19 +708,6 @@ To find objects for a mutation, arguments need to be specified. As with [resolvers](#resolvers), prefer using internal ID or, if needed, a global ID rather than the database ID. -### Fields - -In the most common situations, a mutation would return 2 fields: - -- The resource being modified -- A list of errors explaining why the action could not be - performed. If the mutation succeeded, this list would be empty. - -By inheriting any new mutations from `Mutations::BaseMutation` the -`errors` field is automatically added. A `clientMutationId` field is -also added, this can be used by the client to identify the result of a -single mutation when multiple are performed within a single request. - ### Building Mutations Mutations live in `app/graphql/mutations` ideally grouped per @@ -633,12 +715,16 @@ resources they are mutating, similar to our services. They should inherit `Mutations::BaseMutation`. The fields defined on the mutation will be returned as the result of the mutation. +### Naming conventions + Always provide a consistent GraphQL-name to the mutation, this name is used to generate the input types and the field the mutation is mounted on. The name should look like `<Resource being modified><Mutation class name>`, for example the `Mutations::MergeRequests::SetWip` mutation has GraphQL name `MergeRequestSetWip`. +### Arguments + Arguments required by the mutation can be defined as arguments required for a field. These will be wrapped up in an input type for the mutation. For example, the `Mutations::MergeRequests::SetWip` @@ -667,11 +753,28 @@ This would automatically generate an input type called `clientMutationId`. These arguments are then passed to the `resolve` method of a mutation -as keyword arguments. From here, we can call the service that will -modify the resource. +as keyword arguments. + +### Fields + +In the most common situations, a mutation would return 2 fields: + +- The resource being modified +- A list of errors explaining why the action could not be + performed. If the mutation succeeded, this list would be empty. + +By inheriting any new mutations from `Mutations::BaseMutation` the +`errors` field is automatically added. A `clientMutationId` field is +also added, this can be used by the client to identify the result of a +single mutation when multiple are performed within a single request. + +### The `resolve` method + +The `resolve` method receives the mutation's arguments as keyword arguments. +From here, we can call the service that will modify the resource. The `resolve` method should then return a hash with the same field -names as defined on the mutation and an `errors` array. For example, +names as defined on the mutation including an `errors` array. For example, the `Mutations::MergeRequests::SetWip` defines a `merge_request` field: @@ -690,12 +793,15 @@ should look like this: # The merge request modified, this will be wrapped in the type # defined on the field merge_request: merge_request, - # An array if strings if the mutation failed after authorization - errors: merge_request.errors.full_messages + # An array of strings if the mutation failed after authorization. + # The `errors_on_object` helper collects `errors.full_messages` + errors: errors_on_object(merge_request) } ``` -To make the mutation available it should be defined on the mutation +### Mounting the mutation + +To make the mutation available it must be defined on the mutation type that lives in `graphql/types/mutation_types`. The `mount_mutation` helper method will define a field based on the GraphQL-name of the mutation: @@ -744,6 +850,131 @@ found, we should raise a `Gitlab::Graphql::Errors::ResourceNotAvailable` error. Which will be correctly rendered to the clients. +### Errors in mutations + +We encourage following the practice of [errors as +data](https://graphql-ruby.org/mutations/mutation_errors) for mutations, which +distinguishes errors by who they are relevant to, defined by who can deal with +them. + +Key points: + +- All mutation responses have an `errors` field. This should be populated on + failure, and may be populated on success. +- Consider who needs to see the error: the **user** or the **developer**. +- Clients should always request the `errors` field when performing mutations. +- Errors may be reported to users either at `$root.errors` (top-level error) or at + `$root.data.mutationName.errors` (mutation errors). The location depends on what kind of error + this is, and what information it holds. + +Consider an example mutation `doTheThing` that returns a response with +two fields: `errors: [String]`, and `thing: ThingType`. The specific nature of +the `thing` itself is irrelevant to these examples, as we are considering the +errors. + +There are three states a mutation response can be in: + +- [Success](#success) +- [Failure (relevant to the user)](#failure-relevant-to-the-user) +- [Failure (irrelevant to the user)](#failure-irrelevant-to-the-user) + +#### Success + +In the happy path, errors *may* be returned, along with the anticipated payload, but +if everything was successful, then `errors` should be an empty array, since +there are no problems we need to inform the user of. + +```javascript +{ + data: { + doTheThing: { + errors: [] // if successful, this array will generally be empty. + thing: { .. } + } + } +} +``` + +#### Failure (relevant to the user) + +An error that affects the **user** occurred. We refer to these as _mutation errors_. In +this case there is typically no `thing` to return: + +```javascript +{ + data: { + doTheThing: { + errors: ["you cannot touch the thing"], + thing: null + } + } +} +``` + +Examples of this include: + +- Model validation errors: the user may need to change the inputs. +- Permission errors: the user needs to know they cannot do this, they may need to request permission or sign in. +- Problems with application state that prevent the user's action, for example: merge conflicts, the resource was locked, and so on. + +Ideally, we should prevent the user from getting this far, but if they do, they +need to be told what is wrong, so they understand the reason for the failure and +what they can do to achieve their intent, even if that is as simple as retrying the +request. + +It is possible to return *recoverable* errors alongside mutation data. For example, if +a user uploads 10 files and 3 of them fail and the rest succeed, the errors for the +failures can be made available to the user, alongside the information about +the successes. + +#### Failure (irrelevant to the user) + +One or more *non-recoverable* errors can be returned at the _top level_. These +are things over which the **user** has little to no control, and should mainly +be system or programming problems, that a **developer** needs to know about. +In this case there is no `data`: + +```javascript +{ + errors: [ + {"message": "argument error: expected an integer, got null"}, + ] +} +``` + +This is the result of raising an error during the mutation. In our implementation, +the messages of argument errors and validation errors are returned to the client, and all other +`StandardError` instances are caught, logged and presented to the client with the message set to `"Internal server error"`. +See [`GraphqlController`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/graphql_controller.rb) for details. + +These represent programming errors, such as: + +- A GraphQL syntax error, where an `Int` was passed instead of a `String`, or a required argument was not present. +- Errors in our schema, such as being unable to provide a value for a non-nullable field. +- System errors: for example, a Git storage exception, or database unavailability. + +The user should not be able to cause such errors in regular usage. This category +of errors should be treated as internal, and not shown to the user in specific +detail. + +We need to inform the user when the mutation fails, but we do not need to +tell them why, since they cannot have caused it, and nothing they can do will +fix it, although we may offer to retry the mutation. + +#### Categorizing errors + +When we write mutations, we need to be conscious about which of +these two categories an error state falls into (and communicate about this with +frontend developers to verify our assumptions). This means distinguishing the +needs of the _user_ from the needs of the _client_. + +> _Never catch an error unless the user needs to know about it._ + +If the user does need to know about it, communicate with frontend developers +to make sure the error information we are passing back is useful. + +See also the [frontend GraphQL guide](../development/fe_guide/graphql.md#handling-errors). + ## Validating arguments For validations of single arguments, use the @@ -751,8 +982,8 @@ For validations of single arguments, use the as normal. Sometimes a mutation or resolver may accept a number of optional -arguments, but still want to validate that at least one of the optional -arguments were given. In this situation, consider using the `#ready?` +arguments, but we still want to validate that at least one of the optional +arguments is provided. In this situation, consider using the `#ready?` method within your mutation or resolver to provide the validation. The `#ready?` method will be called before any work is done within the `#resolve` method. @@ -767,7 +998,7 @@ def ready?(**args) end # Always remember to call `#super` - super(args) + super end ``` diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index e9a41e7ec58..6a044004926 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -116,6 +116,73 @@ For instance: - endpoint = expose_path(api_v4_projects_issues_related_merge_requests_path(id: @project.id, issue_iid: @issue.iid)) ``` +## Custom Validators + +In order to validate some parameters in the API request, we validate them +before sending them further (say Gitaly). The following are the +[custom validators](https://GitLab.com/gitlab-org/gitlab/-/tree/master/lib/api/validations/validators), +which we have added so far and how to use them. We also wrote a +guide on how you can add a new custom validator. + +### Using custom validators + +- `FilePath`: + + GitLab supports various functionalities where we need to traverse a file path. + The [`FilePath` validator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/validations/validators/file_path.rb) + validates the parameter value for different cases. Mainly, it checks whether a + path is relative and does it contain `../../` relative traversal using + `File::Separator` or not, and whether the path is absolute, for example + `/etc/passwd/`. + +- `Git SHA`: + + The [`Git SHA` validator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/validations/validators/git_sha.rb) + checks whether the Git SHA parameter is a valid SHA. + It checks by using the regex mentioned in [`commit.rb`](https://gitlab.com/gitlab-org/gitlab/-/commit/b9857d8b662a2dbbf54f46ecdcecb44702affe55#d1c10892daedb4d4dd3d4b12b6d071091eea83df_30_30) file. + +- `Absence`: + + The [`Absence` validator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/validations/validators/absence.rb) + checks whether a particular parameter is absent in a given parameters hash. + +- `IntegerNoneAny`: + + The [`IntegerNoneAny` validator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/validations/validators/integer_none_any.rb) + checks if the value of the given parameter is either an `Integer`, `None`, or `Any`. + It allows only either of these mentioned values to move forward in the request. + +- `ArrayNoneAny`: + + The [`ArrayNoneAny` validator](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/validations/validators/array_none_any.rb) + checks if the value of the given parameter is either an `Array`, `None`, or `Any`. + It allows only either of these mentioned values to move forward in the request. + +### Adding a new custom validator + +Custom validators are a great way to validate parameters before sending +them to platform for further processing. It saves some back-and-forth +from the server to the platform if we identify invalid parameters at the beginning. + +If you need to add a custom validator, it would be added to +it's own file in the [`validators`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/validations/validators) directory. +Since we use [Grape](https://github.com/ruby-grape/grape) to add our API +we inherit from the `Grape::Validations::Base` class in our validator class. +Now, all you have to do is define the `validate_param!` method which takes +in two parameters: the `params` hash and the `param` name to validate. + +The body of the method does the hard work of validating the parameter value +and returns appropriate error messages to the caller method. + +Lastly, we register the validator using the line below: + +```ruby +Grape::Validations.register_validator(<validator name as symbol>, ::API::Helpers::CustomValidators::<YourCustomValidatorClassName>) +``` + +Once you add the validator, make sure you add the `rspec`s for it into +it's own file in the [`validators`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/lib/api/validations/validators) directory. + ## Internal API The [internal API](./internal_api.md) is documented for internal use. Please keep it up to date so we know what endpoints diff --git a/doc/development/application_limits.md b/doc/development/application_limits.md index b13e2994c52..1f7a9ff09b9 100644 --- a/doc/development/application_limits.md +++ b/doc/development/application_limits.md @@ -11,7 +11,7 @@ coordinate with others to [document](../administration/instance_limits.md) and communicate those limits. There is a guide about [introducing application -limits](https://about.gitlab.com/handbook/product/product-management/process/index.html#introducing-application-limits). +limits](https://about.gitlab.com/handbook/product/product-management/process/#introducing-application-limits). ## Development @@ -21,7 +21,7 @@ In the `plan_limits` table, you have to create a new column and insert the limit values. It's recommended to create separate migration script files. 1. Add new column to the `plan_limits` table with non-null default value - that represents desired limit, eg: + that represents desired limit, such as: ```ruby add_column(:plan_limits, :project_hooks, :integer, default: 100, null: false) @@ -31,7 +31,7 @@ limit values. It's recommended to create separate migration script files. enabled. You should use this setting only in special and documented circumstances. 1. (Optionally) Create the database migration that fine-tunes each level with - a desired limit using `create_or_update_plan_limit` migration helper, eg: + a desired limit using `create_or_update_plan_limit` migration helper, such as: ```ruby class InsertProjectHooksPlanLimits < ActiveRecord::Migration[5.2] @@ -65,7 +65,7 @@ for plans that do not exist. #### Get current limit Access to the current limit can be done through the project or the namespace, -eg: +such as: ```ruby project.actual_limits.project_hooks @@ -76,13 +76,13 @@ project.actual_limits.project_hooks There is one method `PlanLimits#exceeded?` to check if the current limit is being exceeded. You can use either an `ActiveRecord` object or an `Integer`. -Ensures that the count of the records does not exceed the defined limit, eg: +Ensures that the count of the records does not exceed the defined limit, such as: ```ruby project.actual_limits.exceeded?(:project_hooks, ProjectHook.where(project: project)) ``` -Ensures that the number does not exceed the defined limit, eg: +Ensures that the number does not exceed the defined limit, such as: ```ruby project.actual_limits.exceeded?(:project_hooks, 10) @@ -115,6 +115,20 @@ it_behaves_like 'includes Limitable concern' do end ``` +### Testing instance-wide limits + +Instance-wide features always use `default` Plan, as instance-wide features +do not have license assigned. + +```ruby +class InstanceVariable + include Limitable + + self.limit_name = 'instance_variables' # Optional as InstanceVariable corresponds with instance_variables + self.limit_scope = Limitable::GLOBAL_SCOPE +end +``` + ### Subscription Plans Self-managed: @@ -123,9 +137,10 @@ Self-managed: GitLab.com: -- `free` - Everyone -- `bronze`- Namespaces with a Bronze subscription -- `silver` - Namespaces with a Silver subscription -- `gold` - Namespaces with a Gold subscription +- `default` - Any system-wide feature +- `free` - Namespaces and projects with a Free subscription +- `bronze`- Namespaces and projects with a Bronze subscription +- `silver` - Namespaces and projects with a Silver subscription +- `gold` - Namespaces and projects with a Gold subscription NOTE: **Note:** The test environment doesn't have any plans. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index f52a5d30c1f..f0ce033587d 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -12,7 +12,7 @@ Both EE and CE require some add-on components called GitLab Shell and Gitaly. Th ## Components -A typical install of GitLab will be on GNU/Linux. It uses NGINX or Apache as a web front end to proxypass the Unicorn web server. By default, communication between Unicorn and the front end is via a Unix domain socket but forwarding requests via TCP is also supported. The web front end accesses `/home/git/gitlab/public` bypassing the Unicorn server to serve static pages, uploads (e.g. avatar images or attachments), and precompiled assets. GitLab serves web pages and the [GitLab API](../api/README.md) using the Unicorn web server. It uses Sidekiq as a job queue which, in turn, uses Redis as a non-persistent database backend for job information, meta data, and incoming jobs. +A typical install of GitLab will be on GNU/Linux. It uses NGINX or Apache as a web front end to proxypass the Unicorn web server. By default, communication between Unicorn and the front end is via a Unix domain socket but forwarding requests via TCP is also supported. The web front end accesses `/home/git/gitlab/public` bypassing the Unicorn server to serve static pages, uploads (e.g. avatar images or attachments), and pre-compiled assets. GitLab serves web pages and the [GitLab API](../api/README.md) using the Unicorn web server. It uses Sidekiq as a job queue which, in turn, uses Redis as a non-persistent database backend for job information, meta data, and incoming jobs. We also support deploying GitLab on Kubernetes using our [GitLab Helm chart](https://docs.gitlab.com/charts/). @@ -196,7 +196,7 @@ GitLab can be considered to have two layers from a process perspective: - Process: `alertmanager` - GitLab.com: [Monitoring of GitLab.com](https://about.gitlab.com/handbook/engineering/monitoring/) -[Alert manager](https://prometheus.io/docs/alerting/alertmanager/) is a tool provided by Prometheus that _"handles alerts sent by client applications such as the Prometheus server. It takes care of deduplicating, grouping, and routing them to the correct receiver integration such as email, PagerDuty, or OpsGenie. It also takes care of silencing and inhibition of alerts."_ You can read more in [issue #45740](https://gitlab.com/gitlab-org/gitlab-foss/issues/45740) about what we will be alerting on. +[Alert manager](https://prometheus.io/docs/alerting/alertmanager/) is a tool provided by Prometheus that _"handles alerts sent by client applications such as the Prometheus server. It takes care of deduplicating, grouping, and routing them to the correct receiver integration such as email, PagerDuty, or OpsGenie. It also takes care of silencing and inhibition of alerts."_ You can read more in [issue #45740](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45740) about what we will be alerting on. #### Certificate management @@ -227,7 +227,7 @@ Consul is a tool for service discovery and configuration. Consul is distributed, - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/database.html#disabling-automatic-database-migration) - [Charts](https://docs.gitlab.com/charts/charts/gitlab/migrations/) - - [Source](../update/upgrading_from_source.md#13-install-libs-migrations-etc) + - [Source](../update/upgrading_from_source.md#13-install-libraries-migrations-etc) - Layer: Core Service (Data) #### Elasticsearch @@ -254,7 +254,7 @@ Elasticsearch is a distributed RESTful search engine built for the cloud. - Process: `gitaly` - GitLab.com: [Service Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#service-architecture) -Gitaly is a service designed by GitLab to remove our need for NFS for Git storage in distributed deployments of GitLab (think GitLab.com or High Availability Deployments). As of 11.3.0, this service handles all Git level access in GitLab. You can read more about the project [in the project's readme](https://gitlab.com/gitlab-org/gitaly). +Gitaly is a service designed by GitLab to remove our need for NFS for Git storage in distributed deployments of GitLab (think GitLab.com or High Availability Deployments). As of 11.3.0, this service handles all Git level access in GitLab. You can read more about the project [in the project's README](https://gitlab.com/gitlab-org/gitaly). #### Praefect @@ -273,7 +273,7 @@ repository updates to secondary nodes. - Configuration: - [Omnibus](../administration/geo/replication/index.md#setup-instructions) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/issues/8) + - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/8) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/geo.md) - Layer: Core Service (Processor) @@ -287,13 +287,13 @@ repository updates to secondary nodes. - Process: `gitlab-exporter` - GitLab.com: [Monitoring of GitLab.com](https://about.gitlab.com/handbook/engineering/monitoring/) -GitLab Exporter is a process designed in house that allows us to export metrics about GitLab application internals to Prometheus. You can read more [in the project's readme](https://gitlab.com/gitlab-org/gitlab-exporter). +GitLab Exporter is a process designed in house that allows us to export metrics about GitLab application internals to Prometheus. You can read more [in the project's README](https://gitlab.com/gitlab-org/gitlab-exporter). #### GitLab Pages - Configuration: - [Omnibus](../administration/pages/index.md) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/issues/37) + - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/37) - [Source](../install/installation.md#install-gitlab-pages) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/pages.md) - Layer: Core Service (Processor) @@ -359,12 +359,12 @@ Grafana is an open source, feature rich metrics dashboard and graph editor for G - [Project page](https://github.com/jaegertracing/jaeger/blob/master/README.md) - Configuration: - - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4104) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/issues/1320) + - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4104) + - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1320) - [Source](../development/distributed_tracing.md#enabling-distributed-tracing) - [GDK](../development/distributed_tracing.md#using-jaeger-in-the-gitlab-development-kit) - Layer: Monitoring -- GitLab.com: [Configuration to enable Tracing for a GitLab instance](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4104) issue. +- GitLab.com: [Configuration to enable Tracing for a GitLab instance](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4104) issue. Jaeger, inspired by Dapper and OpenZipkin, is a distributed tracing system. It can be used for monitoring microservices-based distributed systems. @@ -424,7 +424,7 @@ NGINX has an Ingress port for all HTTP requests and routes them to the appropria - [Project page](https://github.com/prometheus/node_exporter/blob/master/README.md) - Configuration: - [Omnibus](../administration/monitoring/prometheus/node_exporter.md) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/issues/1332) + - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1332) - Layer: Monitoring - Process: `node-exporter` - GitLab.com: [Monitoring of GitLab.com](https://about.gitlab.com/handbook/engineering/monitoring/) @@ -444,7 +444,7 @@ Lightweight connection pooler for PostgreSQL. #### PgBouncer Exporter -- [Project page](https://github.com/stanhu/pgbouncer_exporter/blob/master/README.md) +- [Project page](https://github.com/prometheus-community/pgbouncer_exporter/blob/master/README.md) - Configuration: - [Omnibus](../administration/monitoring/prometheus/pgbouncer_exporter.md) - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#postgresql) @@ -523,7 +523,7 @@ Redis is packaged to provide a place to store: - [Project page](https://github.com/docker/distribution/blob/master/README.md) - Configuration: - - [Omnibus](../update/upgrading_from_source.md#13-install-libs-migrations-etc) + - [Omnibus](../update/upgrading_from_source.md#13-install-libraries-migrations-etc) - [Charts](https://docs.gitlab.com/charts/charts/registry/) - [Source](../administration/packages/container_registry.md#enable-the-container-registry) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/registry.md) @@ -545,13 +545,13 @@ An external registry can also be configured to use GitLab as an auth endpoint. - [Project page](https://github.com/getsentry/sentry/) - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/configuration.html#error-reporting-and-logging-with-sentry) - - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/issues/1319) + - [Charts](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1319) - [Source](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) - [GDK](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) - Layer: Monitoring - GitLab.com: [Searching Sentry](https://about.gitlab.com/handbook/support/workflows/500_errors.html#searching-sentry) -Sentry fundamentally is a service that helps you monitor and fix crashes in realtime. +Sentry fundamentally is a service that helps you monitor and fix crashes in real time. The server is in Python, but it contains a full API for sending events from any language, in any application. For monitoring deployed apps, see the [Sentry integration docs](../user/project/operations/error_tracking.md) @@ -588,7 +588,7 @@ Sidekiq is a Ruby background job processor that pulls jobs from the Redis queue #### LDAP Authentication - Configuration: - - [Omnibus](../administration/auth/ldap.md) + - [Omnibus](../administration/auth/ldap/index.md) - [Charts](https://docs.gitlab.com/charts/charts/globals.html#ldap) - [Source](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/ldap.md) @@ -657,7 +657,7 @@ When making a request to an HTTP Endpoint (think `/users/sign_in`) the request w ### GitLab Git Request Cycle -Below we describe the different pathing that HTTP vs. SSH Git requests will take. There is some overlap with the Web Request Cycle but also some differences. +Below we describe the different paths that HTTP vs. SSH Git requests will take. There is some overlap with the Web Request Cycle but also some differences. ### Web Request (80/443) @@ -790,7 +790,7 @@ ps aux | grep '^git' ``` GitLab has several components to operate. It requires a persistent database -(PostgreSQL) and Redis database, and uses Apache httpd or NGINX to proxypass +(PostgreSQL) and Redis database, and uses Apache `httpd` or NGINX to proxypass Unicorn. All these components should run as different system users to GitLab (e.g., `postgres`, `redis` and `www-data`, instead of `git`). @@ -866,7 +866,7 @@ NGINX: - `/var/log/nginx/` contains error and access logs. -Apache httpd: +Apache `httpd`: - [Explanation of Apache logs](https://httpd.apache.org/docs/2.2/logs.html). - `/var/log/apache2/` contains error and output logs (on Ubuntu). @@ -880,7 +880,7 @@ PostgreSQL: - `/var/log/postgresql/*` -### GitLab specific config files +### GitLab specific configuration files GitLab has configuration files located in `/home/git/gitlab/config/*`. Commonly referenced config files include: @@ -902,7 +902,7 @@ bundle exec rake gitlab:env:info RAILS_ENV=production bundle exec rake gitlab:check RAILS_ENV=production ``` -Note: It is recommended to log into the `git` user using `sudo -i -u git` or `sudo su - git`. While the sudo commands provided by gitlabhq work in Ubuntu they do not always work in RHEL. +Note: It is recommended to log into the `git` user using `sudo -i -u git` or `sudo su - git`. While the sudo commands provided by GitLab work in Ubuntu they do not always work in RHEL. ## GitLab.com diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index 0747224db30..d9e06206961 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -283,7 +283,7 @@ end The final step runs for any un-migrated rows after all of the jobs have been processed. This is in case a Sidekiq process running the background migrations received SIGKILL, leading to the jobs being lost. (See -[more reliable Sidekiq queue](https://gitlab.com/gitlab-org/gitlab-foss/issues/36791) for more information.) +[more reliable Sidekiq queue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/36791) for more information.) If the application does not depend on the data being 100% migrated (for instance, the data is advisory, and not mission-critical), then this final step @@ -312,7 +312,7 @@ to migrate you database down and up, which can result in other background migrations being called. That means that using `spy` test doubles with `have_received` is encouraged, instead of using regular test doubles, because your expectations defined in a `it` block can conflict with what is being -called in RSpec hooks. See [issue #35351](https://gitlab.com/gitlab-org/gitlab/issues/18839) +called in RSpec hooks. See [issue #35351](https://gitlab.com/gitlab-org/gitlab/-/issues/18839) for more details. ## Best practices diff --git a/doc/development/build_test_package.md b/doc/development/build_test_package.md index d478d6e1653..858ff41b685 100644 --- a/doc/development/build_test_package.md +++ b/doc/development/build_test_package.md @@ -1,13 +1,13 @@ # Building a package for testing While developing a new feature or modifying an existing one, it is helpful if an -installable package (or a docker image) containing those changes is available +installable package (or a Docker image) containing those changes is available for testing. For this very purpose, a manual job is provided in the GitLab CI/CD pipeline that can be used to trigger a pipeline in the Omnibus GitLab repository that will create: - A deb package for Ubuntu 16.04, available as a build artifact, and -- A docker image, which is pushed to [Omnibus GitLab's container +- A Docker image, which is pushed to [Omnibus GitLab's container registry](https://gitlab.com/gitlab-org/omnibus-gitlab/container_registry) (images titled `gitlab-ce` and `gitlab-ee` respectively and image tag is the commit which triggered the pipeline). @@ -24,7 +24,7 @@ trigger. If you want to create a package from a specific branch, commit or tag of any of the GitLab components (like GitLab Workhorse, Gitaly, GitLab Pages, etc.), you -can specify the branch name, commit sha or tag in the component's respective +can specify the branch name, commit SHA or tag in the component's respective `*_VERSION` file. For example, if you want to build a package that uses the branch `0-1-stable`, modify the content of `GITALY_SERVER_VERSION` to `0-1-stable` and push the commit. This will create a manual job that can be diff --git a/doc/development/changelog.md b/doc/development/changelog.md index a0e9f9d3be3..5a8e05f888c 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -6,8 +6,8 @@ file, as well as information and history about our changelog process. ## Overview Each bullet point, or **entry**, in our [`CHANGELOG.md`](https://gitlab.com/gitlab-org/gitlab/blob/master/CHANGELOG.md) file is -generated from a single data file in the [`changelogs/unreleased/`](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/changelogs/) -(or corresponding EE) folder. The file is expected to be a [YAML](https://en.wikipedia.org/wiki/YAML) file in the +generated from a single data file in the [`changelogs/unreleased/`](https://gitlab.com/gitlab-org/gitlab/tree/master/changelogs/unreleased/). +The file is expected to be a [YAML](https://en.wikipedia.org/wiki/YAML) file in the following format: ```yaml @@ -282,7 +282,7 @@ multiple times per patch release. This was compounded when we had to release multiple patches at once due to a security issue. We needed to automate all of this manual work. So we -[started brainstorming](https://gitlab.com/gitlab-org/gitlab-foss/issues/17826). +[started brainstorming](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17826). After much discussion we settled on the current solution of one file per entry, and then compiling the entries into the overall `CHANGELOG.md` file during the [release process](https://gitlab.com/gitlab-org/release-tools). diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index db3f58d1d22..42d4b494544 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -10,7 +10,7 @@ the main components.  <!-- Editable diagram available at https://app.diagrams.net/#G1LFl-KW4fgpBPzz8VIH9rsOlAH4t0xwKj --> -On the left side we have the events that can trigger a pipeline based on various events (trigged by a user or automation): +On the left side we have the events that can trigger a pipeline based on various events (triggered by a user or automation): - A `git push` is the most common event that triggers a pipeline. - The [Web API](../../api/pipelines.md#create-a-new-pipeline). @@ -98,7 +98,7 @@ Once all jobs are completed for the current stage, the server "unlocks" all the ### Communication between Runner and GitLab server -Once the Runner is [registered](../../ci/runners/README.md#registering-a-shared-runner) using the registration token, the server knows what type of jobs it can execute. This depends on: +Once the Runner is [registered](https://docs.gitlab.com/runner/register/) using the registration token, the server knows what type of jobs it can execute. This depends on: - The type of runner it is registered as: - a shared runner diff --git a/doc/development/code_comments.md b/doc/development/code_comments.md index a71e2b3c792..b0e83b29cb0 100644 --- a/doc/development/code_comments.md +++ b/doc/development/code_comments.md @@ -9,6 +9,6 @@ Examples: ```ruby # Deprecated scope until code_owner column has been migrated to rule_type. -# To be removed with https://gitlab.com/gitlab-org/gitlab/issues/11834. +# To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/11834. scope :code_owner, -> { where(code_owner: true).or(where(rule_type: :code_owner)) } ``` diff --git a/doc/development/code_review.md b/doc/development/code_review.md index a5ad7dc0f46..9c6cb1d0237 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -308,12 +308,13 @@ experience, refactors the existing code). Then: - Seek to understand the author's perspective. - If you don't understand a piece of code, _say so_. There's a good chance someone else would be confused by it as well. -- Do prefix your comment with "Not blocking:" if you have a small, - non-mandatory improvement you wish to suggest. This lets the author - know that they can optionally resolve this issue in this merge request - or follow-up at a later stage. +- Ensure the author is clear on what is required from them to address/resolve the suggestion. + - Consider using the [Conventional Comment format](https://conventionalcomments.org#format) to + convey your intent. + - For non-mandatory suggestions, decorate with (non-blocking) so the author knows they can + optionally resolve within the merge request or follow-up at a later stage. - After a round of line notes, it can be helpful to post a summary note such as - "LGTM :thumbsup:", or "Just a couple things to address." + "Looks good to me", or "Just a couple things to address." - Assign the merge request to the author if changes are required following your review. @@ -357,8 +358,8 @@ When ready to merge: - If the **latest [Pipeline for Merged Results](../ci/merge_request_pipelines/pipelines_for_merged_results/#pipelines-for-merged-results-premium)** finished less than 2 hours ago, you might merge without starting a new pipeline as the merge request is close enough to `master`. - - If the merge request is from a fork, check how far behind `master` the - source branch is. If it's more than 100 commits behind, ask the author to + - If the **merge request is from a fork**, we can't use [Pipelines for Merged Results](../ci/merge_request_pipelines/pipelines_for_merged_results/index.md#prerequisites), therefore, they're more prone to breaking `master`. + Check how far behind `master` the source branch is. If it's more than 100 commits behind, ask the author to rebase it before merging. - If [master is broken](https://about.gitlab.com/handbook/engineering/workflow/#broken-master), in addition to the two above rules, check that any failure also happens @@ -380,7 +381,7 @@ Merge Results against the latest `master` at the time of the pipeline creation. One of the most difficult things during code review is finding the right balance in how deep the reviewer can interfere with the code created by a -reviewee. +author. - Learning how to find the right balance takes time; that is why we have reviewers that become maintainers after some time spent on reviewing merge @@ -388,7 +389,7 @@ reviewee. - Finding bugs and improving code style is important, but thinking about good design is important as well. Building abstractions and good design is what makes it possible to hide complexity and makes future changes easier. -- Asking the reviewee to change the design sometimes means the complete rewrite +- Asking the author to change the design sometimes means the complete rewrite of the contributed code. It's usually a good idea to ask another maintainer or reviewer before doing it, but have the courage to do it when you believe it is important. @@ -401,7 +402,7 @@ reviewee. - There is a difference in doing things right and doing things right now. Ideally, we should do the former, but in the real world we need the latter as well. A good example is a security fix which should be released as soon as - possible. Asking the reviewee to do the major refactoring in the merge + possible. Asking the author to do the major refactoring in the merge request that is an urgent fix should be avoided. - Doing things well today is usually better than doing something perfectly tomorrow. Shipping a kludge today is usually worse than doing something well @@ -501,7 +502,7 @@ Properties of customer critical merge requests: - The DRI will assign the `customer-critical-merge-request` label to the merge request. - It is required that the reviewer(s) and maintainer(s) involved with a customer critical merge request are engaged as soon as this decision is made. - It is required to prioritize work for those involved on a customer critical merge request so that they have the time available necessary to focus on it. -- It is required to adhere to GitLab [values](https://about.gitlab.com/handbook/values.md) and processes when working on customer critical merge requests, taking particular note of family and friends first/work second, definition of done, iteration, and release when it's ready. +- It is required to adhere to GitLab [values](https://about.gitlab.com/handbook/values/) and processes when working on customer critical merge requests, taking particular note of family and friends first/work second, definition of done, iteration, and release when it's ready. - Customer critical merge requests are required to not reduce security, introduce data-loss risk, reduce availability, nor break existing functionality per the process for [prioritizing technical decisions](https://about.gitlab.com/handbook/engineering/#prioritizing-technical-decisions.md). - On customer critical requests, it is _recommended_ that those involved _consider_ coordinating synchronously (Zoom, Slack) in addition to asynchronously (merge requests comments) if they believe this will reduce elapsed time to merge even though this _may_ sacrifice [efficiency](https://about.gitlab.com/company/culture/all-remote/asynchronous/#evaluating-efficiency.md). - After a customer critical merge request is merged, a retrospective must be completed with the intention of reducing the frequency of future customer critical merge requests. diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index aebff633f58..cea9043a333 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -6,8 +6,8 @@ to contribute to GitLab in a way that is easy for everyone. For a first-time step-by-step guide to the contribution process, see our [Contributing to GitLab](https://about.gitlab.com/community/contribute/) page. -Looking for something to work on? Look for issues with the label -[`~Accepting merge requests`](#how-to-contribute). +Looking for something to work on? See the +[How to contribute](#how-to-contribute) section for more information. GitLab comes in two flavors: @@ -76,9 +76,12 @@ Sign up for the mailing list, answer GitLab questions on StackOverflow or respon ## How to contribute -If you want to contribute to GitLab, -[issues with the `~Accepting merge requests` label](issue_workflow.md#label-for-community-contributors) -are a great place to start. +If you would like to contribute to GitLab: + +- Issues with the + [`~Accepting merge requests` label](issue_workflow.md#label-for-community-contributors) + are a great place to start. +- Consult the [Contribution Flow](#contribution-flow) section to learn the process. If you have any questions or need help visit [Getting Help](https://about.gitlab.com/get-help/) to learn how to communicate with GitLab. We have a [Gitter channel for contributors](https://gitter.im/gitlab/contributors), @@ -96,25 +99,77 @@ a Merge Request. For more information, see the [`gitlab-development-kit`](https://gitlab.com/gitlab-org/gitlab-development-kit) project. -## Contribution Flow - -When contributing to GitLab, your merge request is subject to review by merge request maintainers of a particular specialty. - -When you submit code to GitLab, we really want it to get merged, but there will be times when it will not be merged. - -When maintainers are reading through a merge request they may request guidance from other maintainers. If merge request maintainers conclude that the code should not be merged, our reasons will be fully disclosed. If it has been decided that the code quality is not up to GitLab’s standards, the merge request maintainer will refer the author to our docs and code style guides, and provide some guidance. - -Sometimes style guides will be followed but the code will lack structural integrity, or the maintainer will have reservations about the code’s overall quality. When there is a reservation the maintainer will inform the author and provide some guidance. The author may then choose to update the merge request. Once the merge request has been updated and reassigned to the maintainer, they will review the code again. Once the code has been resubmitted any number of times, the maintainer may choose to close the merge request with a summary of why it will not be merged, as well as some guidance. If the merge request is closed the maintainer will be open to discussion as to how to improve the code so it can be approved in the future. - -GitLab will do its best to review community contributions as quickly as possible. Specially appointed developers review community contributions daily. You may take a look at the [team page](https://about.gitlab.com/company/team/) for the merge request coach who specializes in the type of code you have written and mention them in the merge request. For example, if you have written some JavaScript in your code then you should mention the frontend merge request coach. If your code has multiple disciplines you may mention multiple merge request coaches. - -GitLab receives a lot of community contributions, so if your code has not been reviewed within two days (excluding weekend and public holidays) of its initial submission feel free to re-mention the appropriate merge request coach. - -When submitting code to GitLab, you may feel that your contribution requires the aid of an external library. If your code includes an external library please provide a link to the library, as well as reasons for including it. - -When your code contains more than 500 changes, any major breaking changes, or an external library, `@mention` a maintainer in the merge request. If you are not sure who to mention, the reviewer will add one early in the merge request process. - -### Issues workflow +### Contribution flow + +The general flow of contributing to GitLab is: + +1. [Create a fork](../../user/project/repository/forking_workflow.md#creating-a-fork) + of GitLab. In some cases, you will want to set up the + [GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit) to + [develop against your fork](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/index.md#develop-in-your-own-gitlab-fork). +1. Make your changes in your fork. +1. When you're ready, [create a new merge request](../../user/project/merge_requests/creating_merge_requests.md). +1. In the merge request's description: + - Ensure you provide complete and accurate information. + - Review the provided checklist. +1. Assign the merge request (if possible) to, or `@mention`, one of the + [code owners](../../user/project/code_owners.md) for the relevant project, + and explain that you are ready for review. + +When you submit code to GitLab, we really want it to get merged! However, we always review +submissions carefully, and this takes time. Code submissions will usually be reviewed by two +[domain experts](../code_review.md#domain-experts) before being merged: + +- A [reviewer](../code_review.md#the-responsibility-of-the-reviewer). +- A [maintainer](../code_review.md#the-responsibility-of-the-maintainer). + +Keep the following in mind when submitting merge requests: + +- When reviewers are reading through a merge request they may request guidance from other + reviewers. +- If the code quality is found to not meet GitLab’s standards, the merge request reviewer will + provide guidance and refer the author to our: + - [Documentation](../documentation/styleguide.md) style guide. + - Code style guides. +- Sometimes style guides will be followed but the code will lack structural integrity, or the + reviewer will have reservations about the code’s overall quality. When there is a reservation, + the reviewer will inform the author and provide some guidance. +- Though GitLab generally allows anyone to indicate + [approval](../../user/project/merge_requests/merge_request_approvals.md) of merge requests, the + maintainer may require [approvals from certain reviewers](../code_review.md#approval-guidelines) + before merging a merge request. +- After review, the author may be asked to update the merge request. Once the merge request has been + updated and reassigned to the reviewer, they will review the code again. This process may repeat + any number of times before merge, to help make the contribution the best it can be. + +Sometimes a maintainer may choose to close a merge request. They will fully disclose why it will not +be merged, as well as some guidance. The maintainers will be open to discussion about how to change +the code so it can be approved and merged in the future. + +GitLab will do its best to review community contributions as quickly as possible. Specially +appointed developers review community contributions daily. Look at the +[team page](https://about.gitlab.com/company/team/) for the merge request coach who specializes in +the type of code you have written and mention them in the merge request. For example, if you have +written some front-end code, you should `@mention` the frontend merge request coach. If +your code has multiple disciplines, you may `@mention` multiple merge request coaches. + +GitLab receives a lot of community contributions. If your code has not been reviewed within two +working days of its initial submission, feel free to `@mention` all merge request coaches with +`@gitlab-org/coaches` to get their attention. + +When submitting code to GitLab, you may feel that your contribution requires the aid of an external +library. If your code includes an external library, please provide a link to the library, as well as +reasons for including it. + +`@mention` a maintainer in merge requests that contain: + +- More than 500 changes. +- Any major breaking changes. +- External libraries. + +If you are not sure who to mention, the reviewer will do this for you early in the merge request process. + +#### Issues workflow This [documentation](issue_workflow.md) outlines the current issue workflow: @@ -127,7 +182,7 @@ This [documentation](issue_workflow.md) outlines the current issue workflow: - [Technical and UX debt](issue_workflow.md#technical-and-ux-debt) - [Technical debt in follow-up issues](issue_workflow.md#technical-debt-in-follow-up-issues) -### Merge requests workflow +#### Merge requests workflow This [documentation](merge_request_workflow.md) outlines the current merge request process. diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index be416bf636e..f70299cbfc2 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -2,7 +2,7 @@ ## Issue tracker guidelines -**[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/issues)** for similar entries before +**[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues)** for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an award emoji and/or join the discussion. @@ -16,7 +16,7 @@ see fit. Our issue triage policies are [described in our handbook](https://about.gitlab.com/handbook/engineering/quality/issue-triage/). You are very welcome to help the GitLab team triage issues. -We also organize [issue bash events](https://gitlab.com/gitlab-org/gitlab-foss/issues/17815) +We also organize [issue bash events](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17815) once every quarter. The most important thing is making sure valid issues receive feedback from the @@ -351,15 +351,15 @@ features from GitLab EE to GitLab CE, related issues would be labeled with ~"stewardship". A recent example of this was the issue for -[bringing the time tracking API to GitLab CE](https://gitlab.com/gitlab-org/gitlab-foss/issues/25517#note_20019084). +[bringing the time tracking API to GitLab CE](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/25517#note_20019084). ## Feature proposals To create a feature proposal, open an issue on the -[issue tracker](https://gitlab.com/gitlab-org/gitlab/issues). +[issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues). In order to help track the feature proposals, we have created a -[`feature`](https://gitlab.com/gitlab-org/gitlab/issues?label_name=feature) label. For the time being, users that are not members +[`feature`](https://gitlab.com/gitlab-org/gitlab/-/issues?label_name=feature) label. For the time being, users that are not members of the project cannot add labels. You can instead ask one of the [core team](https://about.gitlab.com/community/core-team/) members to add the label ~feature to the issue or add the following code snippet right after your description in a new line: `~feature`. @@ -403,7 +403,7 @@ below will make it easy to manage this, without unnecessary overhead. Every monthly release has a corresponding issue on the CE issue tracker to keep track of functionality broken by that release and any fixes that need to be included in a patch release (see -[8.3 Regressions](https://gitlab.com/gitlab-org/gitlab-foss/issues/4127) as an example). +[8.3 Regressions](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/4127) as an example). As outlined in the issue description, the intended workflow is to post one note with a reference to an issue describing the regression, and then to update that @@ -420,7 +420,7 @@ in the regression issue as fixes are addressed. ## Technical and UX debt In order to track things that can be improved in GitLab's codebase, -we use the ~"technical debt" label in [GitLab's issue tracker](https://gitlab.com/gitlab-org/gitlab/issues). +we use the ~"technical debt" label in [GitLab's issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues). For missed user experience requirements, we use the ~"UX debt" label. These labels should be added to issues that describe things that can be improved, diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index a70fadfe8a9..14c6b0b1073 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -221,7 +221,7 @@ requirements. 1. [Changelog entry added](../changelog.md), if necessary. 1. Reviewed by relevant (UX/FE/BE/tech writing) reviewers and all concerns are addressed. 1. Merged by a project maintainer. -1. Create an issue in the [infrastructure issue tracker](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues) to inform the Infrastructure department when your contribution is changing default settings or introduces a new setting, if relevant. +1. Create an issue in the [infrastructure issue tracker](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues) to inform the Infrastructure department when your contribution is changing default settings or introduces a new setting, if relevant. 1. Confirmed to be working in the [Canary stage](https://about.gitlab.com/handbook/engineering/#canary-testing) or on GitLab.com once the contribution is deployed. 1. Added to the [release post](https://about.gitlab.com/handbook/marketing/blog/release-posts/), if relevant. diff --git a/doc/development/creating_enums.md b/doc/development/creating_enums.md index e2ebad538d9..3833f771bb5 100644 --- a/doc/development/creating_enums.md +++ b/doc/development/creating_enums.md @@ -83,7 +83,7 @@ module EE end ``` -This looks working as a workaround, however, this approach has some donwside that: +This looks working as a workaround, however, this approach has some downsides that: - Features could move from EE to FOSS or vice versa. Therefore, the offset might be mixed between FOSS and EE in the future. e.g. When you move `activity_limit_exceeded` to FOSS, you'll see `{ unknown_failure: 0, config_error: 1, activity_limit_exceeded: 1_000 }`. diff --git a/doc/development/database/index.md b/doc/development/database/index.md new file mode 100644 index 00000000000..665af623059 --- /dev/null +++ b/doc/development/database/index.md @@ -0,0 +1,48 @@ +# Database guides + +## Tooling + +- [Understanding EXPLAIN plans](../understanding_explain_plans.md) +- [explain.depesz.com](https://explain.depesz.com/) or [explain.dalibo.com](https://explain.dalibo.com/) for visualizing the output of `EXPLAIN` +- [pgFormatter](http://sqlformat.darold.net/) a PostgreSQL SQL syntax beautifier + +## Migrations + +- [What requires downtime?](../what_requires_downtime.md) +- [SQL guidelines](../sql.md) for working with SQL queries +- [Migrations style guide](../migration_style_guide.md) for creating safe SQL migrations +- [Testing Rails migrations](../testing_guide/testing_migrations_guide.md) guide +- [Post deployment migrations](../post_deployment_migrations.md) +- [Background migrations](../background_migrations.md) +- [Swapping tables](../swapping_tables.md) +- [Deleting migrations](../deleting_migrations.md) + +## Debugging + +- Tracing the source of an SQL query using query comments with [Marginalia](../database_query_comments.md) +- Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs) + +## Best practices + +- [Adding database indexes](../adding_database_indexes.md) +- [Foreign keys & associations](../foreign_keys.md) +- [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md) +- [`NOT NULL` constraints](not_null_constraints.md) +- [Strings and the Text data type](strings_and_the_text_data_type.md) +- [Single table inheritance](../single_table_inheritance.md) +- [Polymorphic associations](../polymorphic_associations.md) +- [Serializing data](../serializing_data.md) +- [Hash indexes](../hash_indexes.md) +- [Storing SHA1 hashes as binary](../sha1_as_binary.md) +- [Iterating tables in batches](../iterating_tables_in_batches.md) +- [Insert into tables in batches](../insert_into_tables_in_batches.md) +- [Ordering table columns](../ordering_table_columns.md) +- [Verifying database capabilities](../verifying_database_capabilities.md) +- [Database Debugging and Troubleshooting](../database_debugging.md) +- [Query Count Limits](../query_count_limits.md) +- [Creating enums](../creating_enums.md) + +## Case studies + +- [Database case study: Filtering by label](../filtering_by_label.md) +- [Database case study: Namespaces storage statistics](../namespaces_storage_statistics.md) diff --git a/doc/development/database/not_null_constraints.md b/doc/development/database/not_null_constraints.md new file mode 100644 index 00000000000..e4dec2afa10 --- /dev/null +++ b/doc/development/database/not_null_constraints.md @@ -0,0 +1,217 @@ +# `NOT NULL` constraints + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38358) in GitLab 13.0. + +All attributes that should not have `NULL` as a value, should be defined as `NOT NULL` +columns in the database. + +Depending on the application logic, `NOT NULL` columns should either have a `presence: true` +validation defined in their Model or have a default value as part of their database definition. +As an example, the latter can be true for boolean attributes that should always have a non-`NULL` +value, but have a well defined default value that the application does not need to enforce each +time (for example, `active=true`). + +## Create a new table with `NOT NULL` columns + +When adding a new table, all `NOT NULL` columns should be defined as such directly inside `create_table`. + +For example, consider a migration that creates a table with two `NOT NULL` columns, +`db/migrate/20200401000001_create_db_guides.rb`: + +```ruby +class CreateDbGuides < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + create_table :db_guides do |t| + t.bigint :stars, default: 0, null: false + t.bigint :guide, null: false + end + end +end +``` + +## Add a `NOT NULL` column to an existing table + +With PostgreSQL 11 being the minimum version since GitLab 13.0, adding columns with `NULL` and/or +default values has become much easier and the standard `add_column` helper should be used in all cases. + +For example, consider a migration that adds a new `NOT NULL` column `active` to table `db_guides`, +`db/migrate/20200501000001_add_active_to_db_guides.rb`: + +```ruby +class AddExtendedTitleToSprints < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + add_column :db_guides, :active, :boolean, default: true, null: false + end +end +``` + +## Add a `NOT NULL` constraint to an existing column + +Adding `NOT NULL` to existing database columns requires multiple steps split into at least two +different releases: + +1. Release `N.M` (current release) + + - Ensure the constraint is enforced at the application level (i.e. add a model validation). + - Add a post-deployment migration to add the `NOT NULL` constraint with `validate: false`. + - Add a post-deployment migration to fix the existing records. + + NOTE: **Note:** + Depending on the size of the table, a background migration for cleanup could be required in the next release. + See the [`NOT NULL` constraints on large tables](not_null_constraints.md#not-null-constraints-on-large-tables) section for more information. + + - Create an issue for the next milestone to validate the `NOT NULL` constraint. + +1. Release `N.M+1` (next release) + + - Validate the `NOT NULL` constraint using a post-deployment migration. + +### Example + +Considering a given release milestone, such as 13.0, a model validation has been added into `epic.rb` +to require a description: + +```ruby +class Epic < ApplicationRecord + validates :description, presence: true +end +``` + +The same constraint should be added at the database level for consistency purposes. +We only want to enforce the `NOT NULL` constraint without setting a default, as we have decided +that all epics should have a user-generated description. + +After checking our production database, we know that there are `epics` with `NULL` descriptions, +so we can not add and validate the constraint in one step. + +NOTE: **Note:** +Even if we did not have any epic with a `NULL` description, another instance of GitLab could have +such records, so we would follow the same process either way. + +#### Prevent new invalid records (current release) + +We first add the `NOT NULL` constraint with a `NOT VALID` parameter, which enforces consistency +when new records are inserted or current records are updated. + +In the example above, the existing epics with a `NULL` description will not be affected and you'll +still be able to update records in the `epics` table. However, when you try to update or insert +an epic without providing a description, the constraint causes a database error. + +Adding or removing a `NOT NULL` clause requires that any application changes are deployed _first_. +Thus, adding a `NOT NULL` constraint to an existing column should happen in a post-deployment migration. + +Still in our example, for the 13.0 milestone example (current), we add the `NOT NULL` constraint +with `validate: false` in a post-deployment migration, +`db/post_migrate/20200501000001_add_not_null_constraint_to_epics_description.rb`: + +```ruby +class AddNotNullConstraintToEpicsDescription < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + disable_ddl_transaction! + + def up + # This will add the `NOT NULL` constraint WITHOUT validating it + add_not_null_constraint :epics, :description, validate: false + end + + def down + # Down is required as `add_not_null_constraint` is not reversible + remove_not_null_constraint :epics, :description + end +end +``` + +#### Data migration to fix existing records (current release) + +The approach here depends on the data volume and the cleanup strategy. The number of records that +must be fixed on GitLab.com is a nice indicator that will help us decide whether to use a +post-deployment migration or a background data migration: + +- If the data volume is less than `1000` records, then the data migration can be executed within the post-migration. +- If the data volume is higher than `1000` records, it's advised to create a background migration. + +When unsure about which option to use, please contact the Database team for advice. + +Back to our example, the epics table is not considerably large nor frequently accessed, +so we are going to add a post-deployment migration for the 13.0 milestone (current), +`db/post_migrate/20200501000002_cleanup_epics_with_null_description.rb`: + +```ruby +class CleanupEpicsWithNullDescription < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + # With BATCH_SIZE=1000 and epics.count=29500 on GitLab.com + # - 30 iterations will be run + # - each requires on average ~150ms + # Expected total run time: ~5 seconds + BATCH_SIZE = 1000 + + disable_ddl_transaction! + + class Epic < ActiveRecord::Base + include EachBatch + + self.table_name = 'epics' + end + + def up + Epic.each_batch(of: BATCH_SIZE) do |relation| + relation. + where('description IS NULL'). + update_all(description: 'No description') + end + end + + def down + # no-op : can't go back to `NULL` without first dropping the `NOT NULL` constraint + end +end +``` + +#### Validate the text limit (next release) + +Validating the `NOT NULL` constraint will scan the whole table and make sure that each record is correct. + +Still in our example, for the 13.1 milestone (next), we run the `validate_not_null_constraint` +migration helper in a final post-deployment migration, +`db/post_migrate/20200601000001_validate_not_null_constraint_on_epics_description.rb`: + +```ruby +class ValidateNotNullConstraintOnEpicsDescription < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_not_null_constraint :epics, :description + end + + def down + # no-op + end +end +``` + +## `NOT NULL` constraints on large tables + +If you have to clean up a text column for a really [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/migration_helpers.rb#L12) +(for example, the `artifacts` in `ci_builds`), your background migration will go on for a while and +it will need an additional [background migration cleaning up](../background_migrations.md#cleaning-up) +in the release after adding the data migration. + +In that rare case you will need 3 releases end-to-end: + +1. Release `N.M` - Add the `NOT NULL` constraint and the background-migration to fix the existing records. +1. Release `N.M+1` - Cleanup the background migration. +1. Release `N.M+2` - Validate the `NOT NULL` constraint. + +For these cases, please consult the database team early in the update cycle. The `NOT NULL` +constraint may not be required or other options could exist that do not affect really large +or frequently accessed tables. diff --git a/doc/development/database/strings_and_the_text_data_type.md b/doc/development/database/strings_and_the_text_data_type.md new file mode 100644 index 00000000000..0e77e3972e0 --- /dev/null +++ b/doc/development/database/strings_and_the_text_data_type.md @@ -0,0 +1,288 @@ +# Strings and the Text data type + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30453) in GitLab 13.0. + +When adding new columns that will be used to store strings or other textual information: + +1. We always use the `text` data type instead of the `string` data type. +1. `text` columns should always have a limit set by using the `add_text_limit` migration helper. + +The `text` data type can not be defined with a limit, so `add_text_limit` is enforcing that by +adding a [check constraint](https://www.postgresql.org/docs/11/ddl-constraints.html) on the +column and then validating it at a followup step. + +## Background info + +The reason we always want to use `text` instead of `string` is that `string` columns have the +disadvantage that if you want to update their limit, you have to run an `ALTER TABLE ...` command. + +While a limit is added, the `ALTER TABLE ...` command requires an `EXCLUSIVE LOCK` on the table, which +is held throughout the process of updating the column and while validating all existing records, a +process that can take a while for large tables. + +On the other hand, texts are [more or less equivalent to strings](https://www.depesz.com/2010/03/02/charx-vs-varcharx-vs-varchar-vs-text/) in PostgreSQL, +while having the additional advantage that adding a limit on an existing column or updating their +limit does not require the very costly `EXCLUSIVE LOCK` to be held throughout the validation phase. +We can start by updating the constraint with the valid option off, which requires an `EXCLUSIVE LOCK` +but only for updating the declaration of the columns. We can then validate it at a later step using +`VALIDATE CONSTRAINT`, which requires only a `SHARE UPDATE EXCLUSIVE LOCK` (only conflicts with other +validations and index creation while it allows reads and writes). + +## Create a new table with text columns + +When adding a new table, the limits for all text columns should be added in the same migration as +the table creation. + +For example, consider a migration that creates a table with two text columns, +`db/migrate/20200401000001_create_db_guides.rb`: + +```ruby +class CreateDbGuides < ActiveRecord::Migration[6.0] + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:db_guides) + create_table :db_guides do |t| + t.bigint :stars, default: 0, null: false + t.text :title + t.text :notes + end + end + + # The following add the constraints and validate them immediately (no data in the table) + add_text_limit :db_guides, :title, 128 + add_text_limit :db_guides, :notes, 1024 + end + + def down + # No need to drop the constraints, drop_table takes care of everything + drop_table :db_guides + end +end +``` + +Adding a check constraint requires an exclusive lock while the `ALTER TABLE` that adds is running. +As we don't want the exclusive lock to be held for the duration of a transaction, `add_text_limit` +must always run in a migration with `disable_ddl_transaction!`. + +Also, note that we have to add a check that the table exists so that the migration can be repeated +in case of a failure. + +## Add a text column to an existing table + +Adding a column to an existing table requires an exclusive lock for that table. Even though that lock +is held for a brief amount of time, the time `add_column` needs to complete its execution can vary +depending on how frequently the table is accessed. For example, acquiring an exclusive lock for a very +frequently accessed table may take minutes in GitLab.com and requires the use of `with_lock_retries`. + +For these reasons, it is advised to add the text limit on a separate migration than the `add_column` one. + +For example, consider a migration that adds a new text column `extended_title` to table `sprints`, +`db/migrate/20200501000001_add_extended_title_to_sprints.rb`: + +```ruby +class AddExtendedTitleToSprints < ActiveRecord::Migration[6.0] + DOWNTIME = false + + # rubocop:disable Migration/AddLimitToTextColumns + # limit is added in 20200501000002_add_text_limit_to_sprints_extended_title + def change + add_column :sprints, :extended_title, :text + end + # rubocop:enable Migration/AddLimitToTextColumns +end +``` + +A second migration should follow the first one with a limit added to `extended_title`, +`db/migrate/20200501000002_add_text_limit_to_sprints_extended_title.rb`: + +```ruby +class AddTextLimitToSprintsExtendedTitle < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_text_limit :sprints, :extended_title, 512 + end + + def down + # Down is required as `add_text_limit` is not reversible + remove_text_limit :sprints, :extended_title + end +end +``` + +## Add a text limit constraint to an existing column + +Adding text limits to existing database columns requires multiple steps split into at least two different releases: + +1. Release `N.M` (current release) + + - Add a post-deployment migration to add the limit to the text column with `validate: false`. + - Add a post-deployment migration to fix the existing records. + + NOTE: **Note:** + Depending on the size of the table, a background migration for cleanup could be required in the next release. + See [text limit constraints on large tables](strings_and_the_text_data_type.md#text-limit-constraints-on-large-tables) for more information. + + - Create an issue for the next milestone to validate the text limit. + +1. Release `N.M+1` (next release) + + - Validate the text limit using a post-deployment migration. + +### Example + +Let's assume we want to add a `1024` limit to `issues.title_html` for a given release milestone, +such as 13.0. + +Issues is a pretty busy and large table with more than 25 million rows, so we don't want to lock all +other processes that try to access it while running the update. + +Also, after checking our production database, we know that there are `issues` with more characters in +their title than the 1024 character limit, so we can not add and validate the constraint in one step. + +NOTE: **Note:** +Even if we did not have any record with a title larger than the provided limit, another +instance of GitLab could have such records, so we would follow the same process either way. + +#### Prevent new invalid records (current release) + +We first add the limit as a `NOT VALID` check constraint to the table, which enforces consistency when +new records are inserted or current records are updated. + +In the example above, the existing issues with more than 1024 characters in their title will not be +affected and you'll be still able to update records in the `issues` table. However, when you'd try +to update the `title_html` with a title that has more than 1024 characters, the constraint causes +a database error. + +Adding or removing a constraint to an existing attribute requires that any application changes are +deployed _first_, [otherwise servers still in the old version of the application may try to update the +attribute with invalid values](../multi_version_compatibility.md#ci-artifact-uploads-were-failing). +For these reasons, `add_text_limit` should run in a post-deployment migration. + +Still in our example, for the 13.0 milestone (current), consider that the following validation +has been added to model `Issue`: + +```ruby +validates :title_html, length: { maximum: 1024 } +``` + +We can also update the database in the same milestone by adding the text limit with `validate: false` +in a post-deployment migration, +`db/post_migrate/20200501000001_add_text_limit_migration.rb`: + +```ruby +class AddTextLimitMigration < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + disable_ddl_transaction! + + def up + # This will add the constraint WITHOUT validating it + add_text_limit :issues, :title_html, 1024, validate: false + end + + def down + # Down is required as `add_text_limit` is not reversible + remove_text_limit :issues, :title_html + end +end +``` + +#### Data migration to fix existing records (current release) + +The approach here depends on the data volume and the cleanup strategy. The number of records that must +be fixed on GitLab.com is a nice indicator that will help us decide whether to use a post-deployment +migration or a background data migration: + +- If the data volume is less than `1,000` records, then the data migration can be executed within the post-migration. +- If the data volume is higher than `1,000` records, it's advised to create a background migration. + +When unsure about which option to use, please contact the Database team for advice. + +Back to our example, the issues table is considerably large and frequently accessed, so we are going +to add a background migration for the 13.0 milestone (current), +`db/post_migrate/20200501000002_schedule_cap_title_length_on_issues.rb`: + +```ruby +class ScheduleCapTitleLengthOnIssues < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + # Info on how many records will be affected on GitLab.com + # time each batch needs to run on average, etc ... + BATCH_SIZE = 5000 + DELAY_INTERVAL = 2.minutes.to_i + + # Background migration will update issues whose title is longer than 1024 limit + ISSUES_BACKGROUND_MIGRATION = 'CapTitleLengthOnIssues'.freeze + + disable_ddl_transaction! + + class Issue < ActiveRecord::Base + include EachBatch + + self.table_name = 'issues' + end + + def up + queue_background_migration_jobs_by_range_at_intervals( + Issue.where('char_length(title_html) > 1024'), + ISSUES_MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE + ) + end + + def down + # no-op : the part of the title_html after the limit is lost forever + end +end +``` + +To keep this guide short, we skipped the definition of the background migration and only +provided a high level example of the post-deployment migration that is used to schedule the batches. +You can find more info on the guide about [background migrations](../background_migrations.md) + +#### Validate the text limit (next release) + +Validating the text limit will scan the whole table and make sure that each record is correct. + +Still in our example, for the 13.1 milestone (next), we run the `validate_text_limit` migration +helper in a final post-deployment migration, +`db/post_migrate/20200601000001_validate_text_limit_migration.rb`: + +```ruby +class ValidateTextLimitMigration < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + disable_ddl_transaction! + + def up + validate_text_limit :issues, :title_html + end + + def down + # no-op + end +end +``` + +## Text limit constraints on large tables + +If you have to clean up a text column for a really [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3) +(for example, the `artifacts` in `ci_builds`), your background migration will go on for a while and +it will need an additional [background migration cleaning up](../background_migrations.md#cleaning-up) +in the release after adding the data migration. + +In that rare case you will need 3 releases end-to-end: + +1. Release `N.M` - Add the text limit and the background migration to fix the existing records. +1. Release `N.M+1` - Cleanup the background migration. +1. Release `N.M+2` - Validate the text limit. diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index 3753baa3c63..629aea5b121 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -70,7 +70,9 @@ Use these instructions for exploring the GitLab database while developing with t 1. **PostgreSQL user to authenticate as**: usually your local username, unless otherwise specified during PostgreSQL installation. 1. **Password of the PostgreSQL user**: the password you set when installing PostgreSQL. 1. **Port number to connect to**: `5432` (default). - 1. **Use an ssl connection?** This depends on your installation. Options are: + 1. <!-- vale gitlab.Spelling = NO --> + **Use an ssl connection?** + <!-- vale gitlab.rulename = NO --> This depends on your installation. Options are: - **Use Secure Connection** - **Standard Connection** (default) 1. **(Optional) The database to connect to**: `gitlabhq_development`. @@ -86,7 +88,7 @@ of the extension documentation. ### `ActiveRecord::PendingMigrationError` with Spring -When running specs with the [Spring preloader](rake_tasks.md#speed-up-tests-rake-tasks-and-migrations), +When running specs with the [Spring pre-loader](rake_tasks.md#speed-up-tests-rake-tasks-and-migrations), the test database can get into a corrupted state. Trying to run the migration or dropping/resetting the test database has no effect. diff --git a/doc/development/database_review.md b/doc/development/database_review.md index aa7ebb3756f..5405a8808f0 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -81,13 +81,20 @@ the following preparations into account. - Ensure the down method reverts the changes in `db/structure.sql`. - Update the migration output whenever you modify the migrations during the review process. - Add tests for the migration in `spec/migrations` if necessary. See [Testing Rails migrations at GitLab](testing_guide/testing_migrations_guide.md) for more details. -- When [high-traffic](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/migration_helpers.rb#L12) tables are involved in the migration, use the [`with_lock_retries`](migration_style_guide.md#retry-mechanism-when-acquiring-database-locks) helper method. Review the relevant [examples in our documentation](migration_style_guide.md#examples) for use cases and solutions. +- When [high-traffic](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3) tables are involved in the migration, use the [`with_lock_retries`](migration_style_guide.md#retry-mechanism-when-acquiring-database-locks) helper method. Review the relevant [examples in our documentation](migration_style_guide.md#examples) for use cases and solutions. - Ensure RuboCop checks are not disabled unless there's a valid reason to. +- When adding an index to a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), +test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slack channel and add the execution time to the MR description: + - Execution time largely varies between `#database-lab` and GitLab.com, but an elevated execution time from `#database-lab` + can give a hint that the execution on GitLab.com will also be considerably high. + - If the execution from `#database-lab` is longer than `1h`, the index should be moved to a [post-migration](post_deployment_migrations.md). + Keep in mind that in this case you may need to split the migration and the application changes in separate releases to ensure the index + will be in place when the code that needs it will be deployed. #### Preparation when adding or modifying queries - Write the raw SQL in the MR description. Preferably formatted - nicely with [sqlformat.darold.net](http://sqlformat.darold.net) or + nicely with [pgFormatter](https://sqlformat.darold.net) or [paste.depesz.com](https://paste.depesz.com). - Include the output of `EXPLAIN (ANALYZE, BUFFERS)` of the relevant queries in the description. If the output is too long, wrap it in @@ -134,6 +141,9 @@ the following preparations into account. - [Check indexes are present for foreign keys](migration_style_guide.md#adding-foreign-key-constraints) - Ensure that migrations execute in a transaction or only contain concurrent index/foreign key helpers (with transactions disabled) + - If an index to a large table is added and its execution time was elevated (more than 1h) on `#database-lab`: + - Ensure it was added in a post-migration. + - Maintainer: After the merge request is merged, notify Release Managers about it on `#f_upcoming_release` Slack channel. - Check consistency with `db/structure.sql` and that migrations are [reversible](migration_style_guide.md#reversibility) - Check queries timing (If any): Queries executed in a migration need to fit comfortably within `15s` - preferably much less than that - on GitLab.com. diff --git a/doc/development/db_dump.md b/doc/development/db_dump.md index bb740d12f7b..4095932e44c 100644 --- a/doc/development/db_dump.md +++ b/doc/development/db_dump.md @@ -2,7 +2,7 @@ Sometimes it is useful to import the database from a production environment into a staging environment for testing. The procedure below assumes you have -SSH+sudo access to both the production environment and the staging VM. +SSH and `sudo` access to both the production environment and the staging VM. **Destroy your staging VM** when you are done with it. It is important to avoid data leaks. @@ -20,7 +20,7 @@ sudo gitlab-ctl stop sidekiq ``` Next, we let the production environment stream a compressed SQL dump to our -local machine via SSH, and redirect this stream to a psql client on the staging +local machine via SSH, and redirect this stream to a `psql` client on the staging VM. ```shell diff --git a/doc/development/deleting_migrations.md b/doc/development/deleting_migrations.md index 3ac039a1692..8aa16710d55 100644 --- a/doc/development/deleting_migrations.md +++ b/doc/development/deleting_migrations.md @@ -5,7 +5,7 @@ the possibility of the migration already been included in past releases or in th Because of it, it's not possible to delete existing migrations, as that could lead to: -- Schema inconsistency, as changes introduced into the database were not rollbacked properly. +- Schema inconsistency, as changes introduced into the database were not rolled back properly. - Leaving a record on the `schema_versions` table, that points out to migration that no longer exists on the codebase. Instead of deleting we can opt for disabling the migration. @@ -22,7 +22,7 @@ Migrations can be disabled if: In order to disable a migration, the following steps apply to all types of migrations: -1. Turn the migration into a noop by removing the code inside `#up`, `#down` +1. Turn the migration into a no-op by removing the code inside `#up`, `#down` or `#perform` methods, and adding `#no-op` comment instead. 1. Add a comment explaining why the code is gone. diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index ae84e38e324..7fc33380aba 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -27,7 +27,7 @@ process boundaries, the correlation ID is injected into the outgoing request. Th the propagation of the correlation ID to each downstream subsystem. Correlation IDs are normally generated in the Rails application in response to -certain webrequests. Some user facing systems don't generate correlation IDs in +certain web requests. Some user facing systems don't generate correlation IDs in response to user requests (for example, Git pushes over SSH). ### Developer guidelines for working with correlation IDs diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index 373c5a4ee7d..f3ce9ce3a83 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -43,10 +43,11 @@ For feature flags disabled by default, if they can be used by end users: - Say that it's disabled by default. - Say whether it's enabled on GitLab.com. +- Say whether it can be enabled or disabled per-project. - Say whether it's recommended for production use. - Document how to enable and disable it. -For example, for a feature disabled by default, disabled on GitLab.com, and +For example, for a feature disabled by default, disabled on GitLab.com, can be enabled or disabled per-project, and not ready for production use: ````markdown @@ -55,6 +56,7 @@ not ready for production use: > - [Introduced](link-to-issue) in GitLab 12.0. > - It's deployed behind a feature flag, disabled by default. > - It's disabled on GitLab.com. +> - It's able to be enabled or disabled per-project > - It's not recommended for production use. > - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#anchor-to-section). **(CORE ONLY)** @@ -65,18 +67,24 @@ not ready for production use: <Feature Name> is under development and not ready for production use. It is deployed behind a feature flag that is **disabled by default**. [GitLab administrators with access to the GitLab Rails console](../path/to/administration/feature_flags.md) -can enable it for your instance. +can enable it for your instance. <Feature Name> can be enabled or disabled per-project To enable it: ```ruby +# Instance-wide Feature.enable(:<feature flag>) +# or by project +Feature.enable(:<feature flag>, Project.find(<project id>)) ``` To disable it: ```ruby +# Instance-wide Feature.disable(:<feature flag>) +# or by project +Feature.disable(:<feature flag>, Project.find(<project id>)) ``` ```` @@ -88,10 +96,11 @@ For features that became enabled by default: - Say that it became enabled by default. - Say whether it's enabled on GitLab.com. +- Say whether it can be enabled or disabled per-project. - Say whether it's recommended for production use. - Document how to disable and enable it. -For example, for a feature initially deployed disabled by default, that became enabled by default, that is enabled on GitLab.com, and ready for production use: +For example, for a feature initially deployed disabled by default, that became enabled by default, that is enabled on GitLab.com, that cannot be enabled or disabled per-project, and ready for production use: ````markdown # Feature Name @@ -100,6 +109,7 @@ For example, for a feature initially deployed disabled by default, that became e > - It was deployed behind a feature flag, disabled by default. > - [Became enabled by default](link-to-issue) on GitLab 12.1. > - It's enabled on GitLab.com. +> - It's not able to be enabled or disabled per-project > - It's recommended for production use. > - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#anchor-to-section). **(CORE ONLY)** @@ -110,7 +120,7 @@ For example, for a feature initially deployed disabled by default, that became e <Feature Name> is under development but ready for production use. It is deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](..path/to/administration/feature_flags.md) -can opt to disable it for your instance. +can opt to disable it for your instance it cannot be enabled or disabled per-project. To disable it: @@ -133,10 +143,11 @@ For features enabled by default: - Say it's enabled by default. - Say whether it's enabled on GitLab.com. +- Say whether it can be enabled or disabled per-project. - Say whether it's recommended for production use. - Document how to disable and enable it. -For example, for a feature enabled by default, enabled on GitLab.com, and ready for production use: +For example, for a feature enabled by default, enabled on GitLab.com, cannot be enabled or disabled per-project, and ready for production use: ````markdown # Feature Name @@ -144,6 +155,7 @@ For example, for a feature enabled by default, enabled on GitLab.com, and ready > - [Introduced](link-to-issue) in GitLab 12.0. > - It's deployed behind a feature flag, enabled by default. > - It's enabled on GitLab.com. +> - It's not able to be enabled or disabled per-project > - It's recommended for production use. > - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#anchor-to-section). **(CORE ONLY)** diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index 256d3f5d86c..f845a6ec592 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -59,7 +59,7 @@ However, anyone can contribute [documentation improvements](improvement-workflow ## Markdown and styles [GitLab docs](https://gitlab.com/gitlab-org/gitlab-docs) uses [GitLab Kramdown](https://gitlab.com/gitlab-org/gitlab_kramdown) -as its Markdown rendering engine. See the [GitLab Markdown Guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/) for a complete Kramdown reference. +as its Markdown rendering engine. See the [GitLab Markdown Guide](https://about.gitlab.com/handbook/markdown-guide/) for a complete Kramdown reference. Adhere to the [Documentation Style Guide](styleguide.md). If a style standard is missing, you are welcome to suggest one via a merge request. @@ -67,6 +67,41 @@ Adhere to the [Documentation Style Guide](styleguide.md). If a style standard is See the [Structure](styleguide.md#structure) section of the [Documentation Style Guide](styleguide.md). +## Metadata + +To provide additional directives or useful information, we add metadata in YAML +format to the beginning of each product documentation page. + +For example, the following metadata would be at the beginning of a product +documentation page whose content is primarily associated with the Audit Events +feature: + +```yaml +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- +``` + +The following list describes the YAML parameters in use: + +- `redirect_to`: The relative path and filename (with an `.md` extension) of the + location to which visitors should be redirected for a moved page. + [Learn more](#changing-document-location). +- `stage`: The [Stage](https://about.gitlab.com/handbook/product/categories/#devops-stages) + to which the majority of the page's content belongs. +- `group`: The [Group](https://about.gitlab.com/company/team/structure/#product-groups) + to which the majority of the page's content belongs. +- `info`: The following line, which provides direction to contributors regarding + how to contact the Technical Writer associated with the page's Stage and + Group: `To determine the technical writer assigned to the Stage/Group + associated with this page, see + https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers` +- `disqus_identifier`: Identifier for Disqus commenting system. Used to keep + comments with a page that's been moved to a new URL. + [Learn more](#redirections-for-pages-with-disqus-comments). + ## Changing document location Changing a document's location requires specific steps to ensure that @@ -133,10 +168,9 @@ Things to note: it in for `workflow/lfs/lfs_administration` and `lfs/lfs_administration` and will print the file and the line where this file is mentioned. You may ask why the two greps. Since [we use relative paths to link to - documentation](styleguide.md#links) - , sometimes it might be useful to search a path deeper. + documentation](styleguide.md#links), sometimes it might be useful to search a path deeper. - The `*.md` extension is not used when a document is linked to GitLab's - built-in help page, that's why we omit it in `git grep`. + built-in help page, which is why we omit it in `git grep`. - Use the checklist on the "Change documentation location" MR description template. ### Redirections for pages with Disqus comments @@ -380,7 +414,7 @@ The following GitLab features are used among others: - [Multi project pipelines](../../ci/multi_project_pipeline_graphs.md) - [Review Apps](../../ci/review_apps/index.md) - [Artifacts](../../ci/yaml/README.md#artifacts) -- [Specific Runner](../../ci/runners/README.md#locking-a-specific-runner-from-being-enabled-for-other-projects) +- [Specific Runner](../../ci/runners/README.md#prevent-a-specific-runner-from-being-enabled-for-other-projects) - [Pipelines for merge requests](../../ci/merge_request_pipelines/index.md) ## Testing @@ -477,7 +511,7 @@ This list does not limit what other linters you can add to your local documentat [markdownlint](https://github.com/DavidAnson/markdownlint) checks that Markdown syntax follows [certain rules](https://github.com/DavidAnson/markdownlint/blob/master/doc/Rules.md#rules), and is used by the [`docs-lint` test](#testing) with a [configuration file](#markdownlint-configuration). -Our [Documentation Style Guide](styleguide.md#markdown) and [Markdown Guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/) +Our [Documentation Style Guide](styleguide.md#markdown) and [Markdown Guide](https://about.gitlab.com/handbook/markdown-guide/) elaborate on which choices must be made when selecting Markdown syntax for GitLab documentation. This tool helps catch deviations from those guidelines. @@ -498,7 +532,9 @@ If you wish to use a different configuration file, use the `-c` flag: markdownlint -c <config-file-name> 'doc/**/*.md' ``` -markdownlint can also be run from within text editors using [plugins/extensions](https://github.com/DavidAnson/markdownlint#related), +##### Run markdownlint in an editor + +markdownlint can also be run as a linter within text editors using [plugins/extensions](https://github.com/DavidAnson/markdownlint#related), such as: - [Sublime Text](https://packagecontrol.io/packages/SublimeLinter-contrib-markdownlint) @@ -524,7 +560,7 @@ four repositories that are the sources for <https://docs.gitlab.com>: By default all rules are enabled, so the configuration file is used to disable unwanted rules, and also to configure optional parameters for enabled rules as needed. You can -also check [the issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/64352) that +also check [the issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64352) that tracked the changes required to implement these rules, and details which rules were on or off when markdownlint was enabled on the docs. @@ -546,11 +582,16 @@ and from GitLab's root directory (where `.vale.ini` is located), run: vale --glob='*.{md}' doc ``` -You can also -[configure the text editor of your choice](https://errata-ai.github.io/vale/#local-use-by-a-single-writer) -to display the results. +Vale's error-level test results [are displayed](#testing) in CI pipelines. + +##### Run Vale in an editor + +You can run Vale as a linter within your text editor of choice, with: + +- The Sublime Text [`SublimeLinter-contrib-vale` plugin](https://packagecontrol.io/packages/SublimeLinter-contrib-vale) +- The Visual Studio Code [`testthedocs.vale` extension](https://marketplace.visualstudio.com/items?itemName=testthedocs.vale) -Vale's test results [are displayed](#testing) in CI pipelines. +We don't use [Vale Server](https://errata-ai.github.io/vale/#using-vale-with-a-text-editor-or-another-third-party-application). ##### Disable a Vale test diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 12190e2cb9e..71020e6054e 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -356,7 +356,7 @@ files. ``` This also allows the nav to be displayed on other -highest-level dirs (`/omnibus/`, `/runner/`, etc), +highest-level directories (`/omnibus/`, `/runner/`, etc), linking them back to `/ee/`. The same logic is applied to all sections (`sec[:section_url]`), diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index 56dd3821b1c..942b202a3ec 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -53,12 +53,12 @@ product, and all together are pulled to generate the docs website: - [GitLab Chart](https://gitlab.com/charts/gitlab/tree/master/doc) NOTE: **Note:** -In September 2019, we [moved towards a single codebase](https://gitlab.com/gitlab-org/gitlab/issues/2952), +In September 2019, we [moved towards a single codebase](https://gitlab.com/gitlab-org/gitlab/-/issues/2952), as such the docs for CE and EE are now identical. For historical reasons and in order not to break any existing links throughout the internet, we still maintain the CE docs (`https://docs.gitlab.com/ce/`), although it is hidden from the website, and is now a symlink to the EE docs. When -[Pages supports redirects](https://gitlab.com/gitlab-org/gitlab-pages/issues/24), +[Pages supports redirects](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/24), we will be able to remove this completely. ## Assets @@ -107,13 +107,13 @@ The pipeline in the `gitlab-docs` project: Once a week on Mondays, a scheduled pipeline runs and rebuilds the Docker images used in various pipeline jobs, like `docs-lint`. The Docker image configuration files are -located at <https://gitlab.com/gitlab-org/gitlab-docs/-/tree/master/dockerfiles>. +located in the [Dockerfiles directory](https://gitlab.com/gitlab-org/gitlab-docs/-/tree/master/dockerfiles). If you need to rebuild the Docker images immediately (must have maintainer level permissions): CAUTION: **Caution** If you change the dockerfile configuration and rebuild the images, you can break the master -pipeline in the main `gitlab` repo as well as in `gitlab-docs`. Create an image with +pipeline in the main `gitlab` repository as well as in `gitlab-docs`. Create an image with a different name first and test it to ensure you do not break the pipelines. 1. In [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs), go to **{rocket}** **CI / CD > Pipelines**. @@ -173,7 +173,7 @@ we reference the array with a symbol (`:versions`). ## Bumping versions of CSS and JavaScript Whenever the custom CSS and JavaScript files under `content/assets/` change, -make sure to bump their version in the frontmatter. This method guarantees that +make sure to bump their version in the front matter. This method guarantees that your changes will take effect by clearing the cache of previous files. Always use Nanoc's way of including those files, do not hardcode them in the @@ -207,22 +207,22 @@ If you don't specify `editor:`, the simple one is used by default. ## Algolia search engine -The docs site uses [Algolia docsearch](https://community.algolia.com/docsearch/) +The docs site uses [Algolia DocSearch](https://community.algolia.com/docsearch/) for its search function. This is how it works: -1. GitLab is a member of the [docsearch program](https://community.algolia.com/docsearch/#join-docsearch-program), +1. GitLab is a member of the [DocSearch program](https://community.algolia.com/docsearch/#join-docsearch-program), which is the free tier of [Algolia](https://www.algolia.com/). 1. Algolia hosts a [DocSearch configuration](https://github.com/algolia/docsearch-configs/blob/master/configs/gitlab.json) for the GitLab docs site, and we've worked together to refine it. -1. That [config](https://community.algolia.com/docsearch/config-file.html) is +1. That [configuration](https://community.algolia.com/docsearch/config-file.html) is parsed by their [crawler](https://community.algolia.com/docsearch/crawler-overview.html) every 24h and [stores](https://community.algolia.com/docsearch/inside-the-engine.html) - the [docsearch index](https://community.algolia.com/docsearch/how-do-we-build-an-index.html) + the [DocSearch index](https://community.algolia.com/docsearch/how-do-we-build-an-index.html) on [Algolia's servers](https://community.algolia.com/docsearch/faq.html#where-is-my-data-hosted%3F). -1. On the docs side, we use a [docsearch layout](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/layouts/docsearch.html) which +1. On the docs side, we use a [DocSearch layout](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/layouts/docsearch.html) which is present on pretty much every page except <https://docs.gitlab.com/search/>, which uses its [own layout](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/layouts/instantsearch.html). In those layouts, - there's a JavaScript snippet which initiates docsearch by using an API key + there's a JavaScript snippet which initiates DocSearch by using an API key and an index name (`gitlab`) that are needed for Algolia to show the results. NOTE: **For GitLab employees:** diff --git a/doc/development/documentation/site_architecture/release_process.md b/doc/development/documentation/site_architecture/release_process.md index 13d6540fa35..d100ab8afa8 100644 --- a/doc/development/documentation/site_architecture/release_process.md +++ b/doc/development/documentation/site_architecture/release_process.md @@ -62,30 +62,15 @@ The single docs version must be created before the release merge request, but this needs to happen when the stable branches for all products have been created. 1. Make sure you're in the root path of the `gitlab-docs` repository. -1. Make sure your `master` is updated: - - ```shell - git pull origin master - ``` - 1. Run the Rake task to create the single version: ```shell ./bin/rake "release:single[12.0]" ``` - A new `Dockerfile.12.0` should have been created and committed to a new branch. - -1. Edit `.gitlab-ci.yml` and replace the `BRANCH_` variables with their respective - upstream stable branch. For example, 12.6 would look like: - - ```yaml - variables: - BRANCH_EE: '12-6-stable-ee' - BRANCH_OMNIBUS: '12-6-stable' - BRANCH_RUNNER: '12-6-stable' - BRANCH_CHARTS: '2-6-stable' - ``` + A new `Dockerfile.12.0` should have been created and `.gitlab-ci.yml` should + have the branches variables updated into a new branch. They will be automatically + committed. 1. Push the newly created branch, but **don't create a merge request**. Once you push, the `image:docker-singe` job will create a new Docker image @@ -106,6 +91,9 @@ Visit `http://localhost:4000/12.0/` to see if everything works correctly. ### 3. Create the release merge request +NOTE: **Note:** +To be [automated](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/750). + Now it's time to create the monthly release merge request that adds the new version and rotates the old one: @@ -161,7 +149,7 @@ versions: 1. Run the Rake task that will create all the respective merge requests needed to update the dropdowns and will be set to automatically be merged when their pipelines succeed. The `release-X-Y` branch needs to be present locally, - otherwise the Rake task will fail: + and you need to have switched to it, otherwise the Rake task will fail: ```shell ./bin/rake release:dropdowns diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index d19383bee27..eadcedfaac0 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -34,7 +34,7 @@ For additional details on each, see the [template for new docs](#template-for-ne below. Note that you can include additional subsections, as appropriate, such as 'How it Works', 'Architecture', -and other logical divisions such as pre- and post-deployment steps. +and other logical divisions such as pre-deployment and post-deployment steps. ## Template for new docs diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 6d146051e13..9a93dbf4f94 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -96,7 +96,7 @@ Having a knowledge base in any form that is separate from the documentation woul All GitLab documentation is written using [Markdown](https://en.wikipedia.org/wiki/Markdown). -The [documentation website](https://docs.gitlab.com) uses GitLab Kramdown as its Markdown rendering engine. For a complete Kramdown reference, see the [GitLab Markdown Kramdown Guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/). +The [documentation website](https://docs.gitlab.com) uses GitLab Kramdown as its Markdown rendering engine. For a complete Kramdown reference, see the [GitLab Markdown Kramdown Guide](https://about.gitlab.com/handbook/markdown-guide/). The [`gitlab-kramdown`](https://gitlab.com/gitlab-org/gitlab_kramdown) Ruby gem will support all [GFM markup](../../user/markdown.md) in the future. That is, @@ -134,6 +134,8 @@ be ignored by markdownlint. In general, product names should follow the exact capitalization of the official names of the products, protocols, and so on. +See [`.markdownlint.json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.markdownlint.json) +for the words tested for proper capitalization in GitLab documentation. Some examples fail if incorrect capitalization is used: @@ -246,13 +248,14 @@ Do not include the same information in multiple places. [Link to a SSOT instead. GitLab documentation should be clear and easy to understand. - Be clear, concise, and stick to the goal of the documentation. -- Write in US English with US grammar. +- Write in US English with US grammar. (Tested in [`British.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/British.yml).) - Use inclusive language. ### Point of view In most cases, it’s appropriate to use the second-person (you, yours) point of view, because it’s friendly and easy to understand. +(Tested in [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml).) <!-- How do we harmonize the second person in Pajamas with our first person plural in our doc guide? --> @@ -272,10 +275,11 @@ because it’s friendly and easy to understand. - [GitLab Features](https://about.gitlab.com/features/). For example, Issue Board, Geo, and Runner. - GitLab [product tiers](https://about.gitlab.com/pricing/). For example, GitLab Core - and GitLab Ultimate. + and GitLab Ultimate. (Tested in [`BadgeCapitalization.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/BadgeCapitalization.yml).) - Third-party products. For example, Prometheus, Kubernetes, and Git. - Methods or methodologies. For example, Continuous Integration, Continuous Deployment, Scrum, and Agile. + (Tested in [`.markdownlint.json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.markdownlint.json).) NOTE: **Note:** Some features are also objects. For example, "GitLab's Merge Requests support X" and @@ -289,6 +293,7 @@ tenses, words, and phrases: - Avoid jargon. - Avoid uncommon words. - Don't write in the first person singular. + (Tested in [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml).) - Instead of "I" or "me," use "we," "you," "us," or "one." - When possible, stay user focused by writing in the second person ("you" or the imperative). @@ -311,6 +316,7 @@ tenses, words, and phrases: - <!-- vale gitlab.LatinTerms = NO --> We discourage use of Latin abbreviations, such as "e.g.," "i.e.," or "etc.," as even native users of English might misunderstand them. + (Tested in [`LatinTerms.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/LatinTerms.yml).) - Instead of "i.e.," use "that is." - Instead of "e.g.," use "for example," "such as," "for instance," or "like." - Instead of "etc.," either use "and so on" or consider editing it out, since @@ -319,6 +325,9 @@ tenses, words, and phrases: - Avoid using the word *currently* when talking about the product or its features. The documentation describes the product as it is, and not as it will be at some indeterminate point in the future. +- Don't use profanity or obscenities. Doing so may negatively affect other + users and contributors, which is contrary to GitLab's value of + [diversity and inclusion](https://about.gitlab.com/handbook/values/#diversity-inclusion). ### Word usage clarifications @@ -331,55 +340,55 @@ tenses, words, and phrases: ### Contractions -- Use common contractions when it helps create a friendly and informal tone, especially in tutorials, instructional documentation, and [UIs](https://design.gitlab.com/content/punctuation/#contractions). - -| Do | Don't | -|----------|-----------| -| it's | it is | -| can't | cannot | -| wouldn't | would not | -| you're | you are | -| you've | you have | -| haven't | have not | -| don't | do not | -| we're | we are | -| that's | that is | -| won't | will not | +- Use common contractions when it helps create a friendly and informal tone, especially in tutorials, instructional documentation, and [UIs](https://design.gitlab.com/content/punctuation/#contractions). (Tested in [`Contractions.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Contractions.yml).) + + | Do | Don't | + |----------|-----------| + | it's | it is | + | can't | cannot | + | wouldn't | would not | + | you're | you are | + | you've | you have | + | haven't | have not | + | don't | do not | + | we're | we are | + | that's | that is | + | won't | will not | - Avoid less common contractions: -| Do | Don't | -|--------------|-------------| -| he would | he'd | -| it will | it'll | -| should have | should've | -| there would | there'd | + | Do | Don't | + |--------------|-------------| + | he would | he'd | + | it will | it'll | + | should have | should've | + | there would | there'd | - Do not use contractions with a proper noun and a verb. For example: -| Do | Don't | -|----------------------|---------------------| -| GitLab is creating X | GitLab's creating X | + | Do | Don't | + |----------------------|---------------------| + | GitLab is creating X | GitLab's creating X | - Do not use contractions when you need to emphasize a negative. For example: -| Do | Don't | -|-----------------------------|----------------------------| -| Do **not** install X with Y | **Don't** install X with Y | + | Do | Don't | + |-----------------------------|----------------------------| + | Do **not** install X with Y | **Don't** install X with Y | - Do not use contractions in reference documentation. For example: -| Do | Don't | -|------------------------------------------|----------------------------| -| Do **not** set a limit greater than 1000 | **Don't** set a limit greater than 1000 | -| For `parameter1`, the default is 10 | For `parameter1`, the default's 10 | + | Do | Don't | + |------------------------------------------|----------------------------| + | Do **not** set a limit greater than 1000 | **Don't** set a limit greater than 1000 | + | For `parameter1`, the default is 10 | For `parameter1`, the default's 10 | - Avoid contractions in error messages. Examples: -| Do | Don't | -|------------------------------------------|----------------------------| -| Requests to localhost are not allowed | Requests to localhost aren't allowed | -| Specified URL cannot be used | Specified URL can't be used | + | Do | Don't | + |------------------------------------------|----------------------------| + | Requests to localhost are not allowed | Requests to localhost aren't allowed | + | Specified URL cannot be used | Specified URL can't be used | <!-- vale on --> @@ -414,9 +423,9 @@ Check specific punctuation rules for [lists](#lists) below. | ---- | ------- | | Always end full sentences with a period. | _For a complete overview, read through this document._| | Always add a space after a period when beginning a new sentence. | _For a complete overview, check this doc. For other references, check out this guide._ | -| Do not use double spaces. | --- | +| Do not use double spaces. (Tested in [`SentenceSpacing.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/SentenceSpacing.yml).) | --- | | Do not use tabs for indentation. Use spaces instead. You can configure your code editor to output spaces instead of tabs when pressing the tab key. | --- | -| Use serial commas ("Oxford commas") before the final 'and/or' in a list. | _You can create new issues, merge requests, and milestones._ | +| Use serial commas ("Oxford commas") before the final 'and/or' in a list. (Tested in [`OxfordComma.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/OxfordComma.yml).) | _You can create new issues, merge requests, and milestones._ | | Always add a space before and after dashes when using it in a sentence (for replacing a comma, for example). | _You should try this - or not._ | | Always use lowercase after a colon. | _Related Issues: a way to create a relationship between issues._ | @@ -439,7 +448,7 @@ cp <your_source_directory> <your_destination_directory> - Always start list items with a capital letter, unless they are parameters or commands that are in backticks, or similar. - Always leave a blank line before and after a list. -- Begin a line with spaces (not tabs) to denote a [nested subitem](#nesting-inside-a-list-item). +- Begin a line with spaces (not tabs) to denote a [nested sub-item](#nesting-inside-a-list-item). ### Ordered vs. unordered lists @@ -599,7 +608,7 @@ that is best described by a matrix, tables are the best choice for use. ### Creation guidelines -Due to accessibility and scanability requirements, tables should not have any +Due to accessibility and scannability requirements, tables should not have any empty cells. If there is no otherwise meaningful value for a cell, consider entering *N/A* (for 'not applicable') or *none*. @@ -648,7 +657,7 @@ For other punctuation rules, please refer to the links shift too, which eventually leads to dead links. If you think it is compelling to add numbers in headings, make sure to at least discuss it with someone in the Merge Request. -- [Avoid using symbols and special chars](https://gitlab.com/gitlab-org/gitlab-docs/issues/84) +- [Avoid using symbols and special characters](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/84) in headers. Whenever possible, they should be plain and short text. - Avoid adding things that show ephemeral statuses. For example, if a feature is considered beta or experimental, put this information in a note, not in the heading. @@ -719,6 +728,7 @@ We include guidance for links in the following categories: - How to set up [criteria](#basic-link-criteria) for configuring a link. - What to set up when [linking to a `help`](../documentation/index.md#linking-to-help) page. - How to set up [links to internal documentation](#links-to-internal-documentation) for cross-references. +- How to set up [links to external documentation](#links-to-external-documentation) for authoritative sources. - When to use [links requiring permissions](#links-requiring-permissions). - How to set up a [link to a video](#link-to-video). - How to [include links with version text](#text-for-documentation-requiring-version-text). @@ -771,6 +781,12 @@ To link to internal documentation: NOTE: **Note**: Using the Markdown extension is necessary for the [`/help`](index.md#gitlab-help) section of GitLab. +### Links to external documentation + +When describing interactions with external software, it's often helpful to include links to external +documentation. When possible, make sure that you are linking to an **authoritative** source. +For example, if you're describing a feature in Microsoft's Active Directory, include a link to official Microsoft documentation. + ### Links requiring permissions Don't link directly to: @@ -793,7 +809,7 @@ Instead: Example: ```markdown -For more information, see the [confidential issue](../../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab-foss/issues/<issue_number>`. +For more information, see the [confidential issue](../../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab-foss/-/issues/<issue_number>`. ``` ### Link to specific lines of code @@ -1040,11 +1056,11 @@ of language classes available. | `xml` | | | `yaml` | Alias: `yml`. | -For a complete reference on code blocks, check the [Kramdown guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/#code-blocks). +For a complete reference on code blocks, check the [Kramdown guide](https://about.gitlab.com/handbook/markdown-guide/#code-blocks). ## GitLab SVG icons -> [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/issues/384) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/384) in GitLab 12.7. You can use icons from the [GitLab SVG library](https://gitlab-org.gitlab.io/gitlab-svgs/) directly in the documentation. @@ -1359,7 +1375,35 @@ versions back, you can consider removing the text if it's irrelevant or confusin For example, if the current major version is 12.x, version text referencing versions of GitLab 8.x and older are candidates for removal if necessary for clearer or cleaner docs. -## Product badges +## Products and features + +Refer to the information in this section when describing products and features +within the GitLab product documentation. + +### Avoid line breaks in names + +When entering a product or feature name that includes a space (such as +GitLab Community Edition) or even other companies' products (such as +Amazon Web Services), be sure to not split the product or feature name across +lines with an inserted line break. Splitting product or feature names across +lines makes searching for these items more difficult, and can cause problems if +names change. + +For example, the followng Markdown content is *not* formatted correctly: + +```markdown +When entering a product or feature name that includes a space (such as GitLab +Community Edition), don't split the product or feature name across lines. +``` + +Instead, it should appear similar to the following: + +```markdown +When entering a product or feature name that includes a space (such as +GitLab Community Edition), don't split the product or feature name across lines. +``` + +### Product badges When a feature is available in EE-only tiers, add the corresponding tier according to the feature availability: @@ -1399,7 +1443,7 @@ For example: The absence of tiers' mentions mean that the feature is available in GitLab Core, GitLab.com Free, and all higher tiers. -### How it works +#### How it works Introduced by [!244](https://gitlab.com/gitlab-org/gitlab-docs/-/merge_requests/244), the special markup `**(STARTER)**` will generate a `span` element to trigger the @@ -1608,7 +1652,7 @@ Rendered example: - Wherever needed use this personal access token: `<your_access_token>`. - Always put the request first. `GET` is the default so you don't have to include it. -- Use double quotes to the URL when it includes additional parameters. +- Wrap the URL in double quotes (`"`). - Prefer to use examples using the personal access token and don't pass data of username and password. @@ -1686,9 +1730,8 @@ Use `%2F` for slashes (`/`). #### Pass arrays to API calls The GitLab API sometimes accepts arrays of strings or integers. For example, to -restrict the sign-up e-mail domains of a GitLab instance to `*.example.com` and -`example.net`, you would do something like this: +exclude specific users when requesting a list of users for a project, you would do something like this: ```shell -curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "domain_whitelist[]=*.example.com" --data "domain_whitelist[]=example.net" https://gitlab.example.com/api/v4/application/settings +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "skip_users[]=<user_id>" --data "skip_users[]=<user_id>" https://gitlab.example.com/api/v4/projects/<project_id>/users ``` diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index ab6200155bf..c3e15cb1b2b 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -54,7 +54,7 @@ To update GitLab documentation: 1. Follow the described standards and processes listed on the page, including: - The [Structure and template](structure.md) page. - The [Style Guide](styleguide.md). - - The [Markdown Guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/). + - The [Markdown Guide](https://about.gitlab.com/handbook/markdown-guide/). 1. Follow GitLab's [Merge Request Guidelines](../contributing/merge_request_workflow.md#merge-request-guidelines). TIP: **Tip:** @@ -104,7 +104,7 @@ The process involves the following: - Ensure the appropriate labels are applied, including any required to pick a merge request into a release. - Ensure that, if there has not been a Technical Writer review completed or scheduled, they - [create the required issue](https://gitlab.com/gitlab-org/gitlab/issues/new?issuable_template=Doc%20Review), assign to the Technical Writer of the given stage group, + [create the required issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Doc%20Review), assign to the Technical Writer of the given stage group, and link it from the merge request. The process is reflected in the **Documentation** @@ -113,14 +113,14 @@ The process is reflected in the **Documentation** ## Other ways to help If you have ideas for further documentation resources please -[create an issue](https://gitlab.com/gitlab-org/gitlab/issues/new?issuable_template=Documentation) +[create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Documentation) using the Documentation template. ## Post-merge reviews If not assigned to a Technical Writer for review prior to merging, a review must be scheduled immediately after merge by the developer or maintainer. For this, -create an issue using the [Doc Review description template](https://gitlab.com/gitlab-org/gitlab/issues/new?issuable_template=Doc%20Review) +create an issue using the [Doc Review description template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Doc%20Review) and link to it from the merged merge request that introduced the documentation change. Circumstances where a regular pre-merge Technical Writer review might be skipped include: diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index 3951b0516e8..e22e96b6f06 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -11,7 +11,7 @@ ## Act as CE when unlicensed Since the implementation of -[GitLab CE features to work with unlicensed EE instance](https://gitlab.com/gitlab-org/gitlab/issues/2500) +[GitLab CE features to work with unlicensed EE instance](https://gitlab.com/gitlab-org/gitlab/-/issues/2500) GitLab Enterprise Edition should work like GitLab Community Edition when no license is active. So EE features always should be guarded by `project.feature_available?` or `group.feature_available?` (or @@ -165,8 +165,6 @@ There are a few gotchas with it: end ``` - This would require updating CE first, or make sure this is back ported to CE. - When prepending, place them in the `ee/` specific sub-directory, and wrap class or module in `module EE` to avoid naming conflicts. diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 185f536fc01..9f54386f1af 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -24,19 +24,19 @@ See the [Elasticsearch GDK setup instructions](https://gitlab.com/gitlab-org/git - `gitlab:elastic:test:index_size`: Tells you how much space the current index is using, as well as how many documents are in the index. - `gitlab:elastic:test:index_size_change`: Outputs index size, reindexes, and outputs index size again. Useful when testing improvements to indexing size. -Additionally, if you need large repos or multiple forks for testing, please consider [following these instructions](rake_tasks.md#extra-project-seed-options) +Additionally, if you need large repositories or multiple forks for testing, please consider [following these instructions](rake_tasks.md#extra-project-seed-options) ## How does it work? -The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a Rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [/ee/app/models/concerns/elastic/application_versioned_search.rb](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb). +The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a Rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [`/ee/app/models/concerns/elastic/application_versioned_search.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb). -After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/topics/data-types#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). +After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/-/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/topics/data-types#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). -Search queries are generated by the concerns found in [ee/app/models/concerns/elastic](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them! +Search queries are generated by the concerns found in [`ee/app/models/concerns/elastic`](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them! ## Existing Analyzers/Tokenizers/Filters -These are all defined in [ee/lib/elastic/latest/config.rb](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/elastic/latest/config.rb) +These are all defined in [`ee/lib/elastic/latest/config.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/elastic/latest/config.rb) ### Analyzers @@ -54,7 +54,7 @@ Please see the `sha_tokenizer` explanation later below for an example. #### `code_analyzer` -Used when indexing a blob's filename and content. Uses the `whitespace` tokenizer and the filters: [`code`](#code), [`edgeNGram_filter`](#edgengram_filter), `lowercase`, and `asciifolding` +Used when indexing a blob's filename and content. Uses the `whitespace` tokenizer and the filters: [`code`](#code), `lowercase`, and `asciifolding` The `whitespace` tokenizer was selected in order to have more control over how tokens are split. For example the string `Foo::bar(4)` needs to generate tokens like `Foo` and `bar(4)` in order to be properly searched. @@ -71,7 +71,7 @@ Not directly used for indexing, but rather used to transform a search input. Use #### `sha_tokenizer` -This is a custom tokenizer that uses the [`edgeNGram` tokenizer](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-edgengram-tokenizer.html) to allow SHAs to be searcheable by any sub-set of it (minimum of 5 chars). +This is a custom tokenizer that uses the [`edgeNGram` tokenizer](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-edgengram-tokenizer.html) to allow SHAs to be searchable by any sub-set of it (minimum of 5 chars). Example: @@ -149,7 +149,7 @@ These proxy objects would talk to Elasticsearch server directly (see top half of  -In the planned new design, each model would have a pair of corresponding subclassed proxy objects, in which model-specific logic is located. For example, `Snippet` would have `SnippetClassProxy` and `SnippetInstanceProxy` (being subclass of `Elasticsearch::Model::Proxy::ClassMethodsProxy` and `Elasticsearch::Model::Proxy::InstanceMethodsProxy`, respectively). +In the planned new design, each model would have a pair of corresponding sub-classed proxy objects, in which model-specific logic is located. For example, `Snippet` would have `SnippetClassProxy` and `SnippetInstanceProxy` (being subclass of `Elasticsearch::Model::Proxy::ClassMethodsProxy` and `Elasticsearch::Model::Proxy::InstanceMethodsProxy`, respectively). `__elasticsearch__` would represent another layer of proxy object, keeping track of multiple actual proxy objects. It would forward method calls to the appropriate index. For example: @@ -175,6 +175,53 @@ If the current version is `v12p1`, and we need to create a new version for `v12p 1. Change the namespace for files under `v12p1` folder from `Latest` to `V12p1` 1. Make changes to files under the `latest` folder as needed +## Performance Monitoring + +### Prometheus + +GitLab exports [Prometheus +metrics](../administration/monitoring/prometheus/gitlab_metrics.md) relating to +the number of requests and timing for all web/API requests and Sidekiq jobs, +which can help diagnose performance trends and compare how Elasticsearch timing +is impacting overall performance relative to the time spent doing other things. + +#### Indexing queues + +GitLab also exports [Prometheus +metrics](../administration/monitoring/prometheus/gitlab_metrics.md) for +indexing queues, which can help diagnose performance bottlenecks and determine +whether or not your GitLab instance or Elasticsearch server can keep up with +the volume of updates. + +### Logs + +All of the indexing happens in Sidekiq, so much of the relevant logs for the +Elasticsearch integration can be found in +[`sidekiq.log`](../administration/logs.md#sidekiqlog). In particular, all +Sidekiq workers that make requests to Elasticsearch in any way will log the +number of requests and time taken querying/writing to Elasticsearch. This can +be useful to understand whether or not your cluster is keeping up with +indexing. + +Searching Elasticsearch is done via ordinary web workers handling requests. Any +requests to load a page or make an API request, which then make requests to +Elasticsearch, will log the number of requests and the time taken to +[`production_json.log`](../administration/logs.md#production_jsonlog). These +logs will also include the time spent on Database and Gitaly requests, which +may help to diagnose which part of the search is performing poorly. + +There are additional logs specific to Elasticsearch that are sent to +[`elasticsearch.log`](../administration/logs.md#elasticsearchlog-starter-only) +that may contain information to help diagnose performance issues. + +### Performance Bar + +Elasticsearch requests will be displayed in the [`Performance +Bar`](../administration/monitoring/performance/performance_bar.md), which can +be used both locally in development and on any deployed GitLab instance to +diagnose poor search performance. This will show the exact queries being made, +which is useful to diagnose why a search might be slow. + ## Troubleshooting ### Getting `flood stage disk watermark [95%] exceeded` diff --git a/doc/development/emails.md b/doc/development/emails.md index e6e2ea8aae7..2477a51f78f 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -1,5 +1,20 @@ # Dealing with email in development +## Ensuring compatibility with mailer Sidekiq jobs + +A Sidekiq job is enqueued whenever `deliver_later` is called on an `ActionMailer`. +If a mailer argument needs to be added or removed, it is important to ensure +both backward and forward compatibility. Adhere to the Sidekiq Style Guide steps for +[changing the arguments for a worker](sidekiq_style_guide.md#changing-the-arguments-for-a-worker). + +In the following example from [`NotificationService`](https://gitlab.com/gitlab-org/gitlab/-/blob/33ccb22e4fc271dbaac94b003a7a1a2915a13441/app/services/notification_service.rb#L74) +adding or removing an argument in this mailer's definition may cause problems +during deployment before all Rails and Sidekiq nodes have the updated code. + +```ruby +mailer.unknown_sign_in_email(user, ip, time).deliver_later +``` + ## Sent emails To view rendered emails "sent" in your development instance, visit diff --git a/doc/development/fe_guide/axios.md b/doc/development/fe_guide/axios.md index f8d301dac5e..38a8c8f1086 100644 --- a/doc/development/fe_guide/axios.md +++ b/doc/development/fe_guide/axios.md @@ -1,15 +1,15 @@ # Axios -We use [axios](https://github.com/axios/axios) to communicate with the server in Vue applications and most new code. +We use [Axios](https://github.com/axios/axios) to communicate with the server in Vue applications and most new code. -In order to guarantee all defaults are set you *should not use `axios` directly*, you should import `axios` from `axios_utils`. +In order to guarantee all defaults are set you *should not use Axios directly*, you should import Axios from `axios_utils`. ## CSRF token -All our request require a CSRF token. -To guarantee this token is set, we are importing [axios](https://github.com/axios/axios), setting the token, and exporting `axios` . +All our requests require a CSRF token. +To guarantee this token is set, we are importing [Axios](https://github.com/axios/axios), setting the token, and exporting `axios` . -This exported module should be used instead of directly using `axios` to ensure the token is set. +This exported module should be used instead of directly using Axios to ensure the token is set. ## Usage @@ -30,7 +30,7 @@ This exported module should be used instead of directly using `axios` to ensure }); ``` -## Mock axios response in tests +## Mock Axios response in tests To help us mock the responses we are using [axios-mock-adapter](https://github.com/ctimmerm/axios-mock-adapter). @@ -41,7 +41,7 @@ Advantages over [`spyOn()`](https://jasmine.github.io/api/edge/global.html#spyOn - simple API to test error cases - provides `replyOnce()` to allow for different responses -We have also decided against using [axios interceptors](https://github.com/axios/axios#interceptors) because they are not suitable for mocking. +We have also decided against using [Axios interceptors](https://github.com/axios/axios#interceptors) because they are not suitable for mocking. ### Example @@ -67,7 +67,7 @@ We have also decided against using [axios interceptors](https://github.com/axios }); ``` -### Mock poll requests in tests with axios +### Mock poll requests in tests with Axios Because polling function requires a header object, we need to always include an object as the third argument: diff --git a/doc/development/fe_guide/dependencies.md b/doc/development/fe_guide/dependencies.md index 0f5825992e9..7f078df887d 100644 --- a/doc/development/fe_guide/dependencies.md +++ b/doc/development/fe_guide/dependencies.md @@ -32,7 +32,7 @@ because they can create conflicts in the dependency tree. Blocked dependencies a ### BootstrapVue -[BootstrapVue](https://bootstrap-vue.js.org/) is a component library built with Vue.js and Bootstrap. +[BootstrapVue](https://bootstrap-vue.org/) is a component library built with Vue.js and Bootstrap. We wrap BootstrapVue components in [GitLab UI](https://gitlab.com/gitlab-org/gitlab-ui/) with the purpose of applying visual styles and usage guidelines specified in the [Pajamas Design System](https://design.gitlab.com/). For this reason, we recommend not installing diff --git a/doc/development/fe_guide/development_process.md b/doc/development/fe_guide/development_process.md index 6e078f097cd..892e931bf5b 100644 --- a/doc/development/fe_guide/development_process.md +++ b/doc/development/fe_guide/development_process.md @@ -23,7 +23,7 @@ Please use your best judgment when to use it and please contribute new points th - [ ] Are all necessary UX specifications available that you will need in order to implement? Are there new UX components/patterns in the designs? Then contact the UI component team early on. How should error messages or validation be handled? - [ ] **Library usage** Use Vuex as soon as you have even a medium state to manage, use Vue router if you need to have different views internally and want to link from the outside. Check what libraries we already have for which occasions. - [ ] **Plan your implementation:** - - [ ] **Architecture plan:** Create a plan aligned with GitLab's architecture, how you are going to do the implementation, for example Vue application setup and its components (through [onion skinning](https://gitlab.com/gitlab-org/gitlab-foss/issues/35873#note_39994091)), Store structure and data flow, which existing Vue components can you reuse. It's a good idea to go through your plan with another engineer to refine it. + - [ ] **Architecture plan:** Create a plan aligned with GitLab's architecture, how you are going to do the implementation, for example Vue application setup and its components (through [onion skinning](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/35873#note_39994091)), Store structure and data flow, which existing Vue components can you reuse. It's a good idea to go through your plan with another engineer to refine it. - [ ] **Backend:** The best way is to kickoff the implementation in a call and discuss with the assigned Backend engineer what you will need from the backend and also when. Can you reuse existing API's? How is the performance with the planned architecture? Maybe create together a JSON mock object to already start with development. - [ ] **Communication:** It also makes sense to have for bigger features an own slack channel (normally called #f_{feature_name}) and even weekly demo calls with all people involved. - [ ] **Dependency Plan:** Are there big dependencies in the plan between you and others, then maybe create an execution diagram to show what is blocking which part and the order of the different parts. diff --git a/doc/development/fe_guide/droplab/plugins/ajax.md b/doc/development/fe_guide/droplab/plugins/ajax.md index abc208e7568..f22d95064dd 100644 --- a/doc/development/fe_guide/droplab/plugins/ajax.md +++ b/doc/development/fe_guide/droplab/plugins/ajax.md @@ -6,7 +6,7 @@ Add the `Ajax` object to the plugins array of a `DropLab.prototype.init` or `DropLab.prototype.addHook` call. -`Ajax` requires 2 config values, the `endpoint` and `method`. +`Ajax` requires 2 configuration values, the `endpoint` and `method`. - `endpoint` should be a URL to the request endpoint. - `method` should be `setData` or `addData`. diff --git a/doc/development/fe_guide/droplab/plugins/filter.md b/doc/development/fe_guide/droplab/plugins/filter.md index 876149e4872..e8194e45a41 100644 --- a/doc/development/fe_guide/droplab/plugins/filter.md +++ b/doc/development/fe_guide/droplab/plugins/filter.md @@ -7,7 +7,7 @@ to the dropdown using a simple fuzzy string search of an input value. Add the `Filter` object to the plugins array of a `DropLab.prototype.init` or `DropLab.prototype.addHook` call. -- `Filter` requires a config value for `template`. +- `Filter` requires a configuration value for `template`. - `template` should be the key of the objects within your data array that you want to compare to the user input string, for filtering. diff --git a/doc/development/fe_guide/droplab/plugins/input_setter.md b/doc/development/fe_guide/droplab/plugins/input_setter.md index 9b2e1e8faab..b873b7a14ee 100644 --- a/doc/development/fe_guide/droplab/plugins/input_setter.md +++ b/doc/development/fe_guide/droplab/plugins/input_setter.md @@ -6,12 +6,12 @@ Add the `InputSetter` object to the plugins array of a `DropLab.prototype.init` or `DropLab.prototype.addHook` call. -- `InputSetter` requires a config value for `input` and `valueAttribute`. +- `InputSetter` requires a configuration value for `input` and `valueAttribute`. - `input` should be the DOM element that you want to manipulate. - `valueAttribute` should be a string that is the name of an attribute on your list items that is used to get the value to update the `input` element with. -You can also set the `InputSetter` config to an array of objects, which will allow you to update multiple elements. +You can also set the `InputSetter` configuration to an array of objects, which will allow you to update multiple elements. ```html <input id="input" value=""> diff --git a/doc/development/fe_guide/emojis.md b/doc/development/fe_guide/emojis.md index 6d324d4c4a0..3cd14c0dfd3 100644 --- a/doc/development/fe_guide/emojis.md +++ b/doc/development/fe_guide/emojis.md @@ -1,6 +1,6 @@ # Emojis -GitLab supports native unicode emojis and fallsback to image-based emojis selectively +GitLab supports native Unicode emojis and falls back to image-based emojis selectively when your platform does not support it. ## How to update Emojis @@ -21,7 +21,7 @@ when your platform does not support it. 1. Ensure you see new individual images copied into `app/assets/images/emoji/` 1. Ensure you can see the new emojis and their aliases in the GFM Autocomplete 1. Ensure you can see the new emojis and their aliases in the award emoji menu - 1. You might need to add new emoji unicode support checks and rules for platforms + 1. You might need to add new emoji Unicode support checks and rules for platforms that do not support a certain emoji and we need to fallback to an image. See `app/assets/javascripts/emoji/support/is_emoji_unicode_supported.js` and `app/assets/javascripts/emoji/support/unicode_support_map.js` diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index 8f8f162609a..4f814f3cdde 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -82,7 +82,7 @@ To avoid this behavior, add the class `js-no-auto-disable` to the button. ### 5. Should I use a full URL (i.e. `gon.gitlab_url`) or a full path (i.e. `gon.relative_url_root`) when referencing backend endpoints? It's preferred to use a **full path** over a **full URL** because the URL will use the hostname configured with -GitLab which may not match the request. This will cause [CORS issues like this Web IDE one](https://gitlab.com/gitlab-org/gitlab/issues/36810). +GitLab which may not match the request. This will cause [CORS issues like this Web IDE one](https://gitlab.com/gitlab-org/gitlab/-/issues/36810). Example: @@ -141,7 +141,7 @@ function initFoo() { }); } -// Vuex action can now reference the path from it's state :) +// Vuex action can now reference the path from its state :) export const fetchFoos = ({ state }) => { return axios.get(state.settings.fooPath); }; diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index caf84d04490..191ebd2ff58 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -1,7 +1,69 @@ # GraphQL +## Getting Started + +### Helpful Resources + +**General resources**: + +- [📚 Official Introduction to GraphQL](https://graphql.org/learn/) +- [📚 Official Introduction to Apollo](https://www.apollographql.com/docs/tutorial/introduction/) + +**GraphQL at GitLab**: + +- [🎬 GitLab Unfiltered GraphQL playlist](https://www.youtube.com/watch?v=wHPKZBDMfxE&list=PL05JrBw4t0KpcjeHjaRMB7IGB2oDWyJzv) +- [🎬 GraphQL at GitLab: Deep Dive](../api_graphql_styleguide.md#deep-dive) (video) by Nick Thomas + - An overview of the history of GraphQL at GitLab (not frontend-specific) +- [🎬 GitLab Feature Walkthrough with GraphQL and Vue Apollo](https://www.youtube.com/watch?v=6yYp2zB7FrM) (video) by Natalia Tepluhina + - A real-life example of implementing a frontend feature in GitLab using GraphQL +- [🎬 History of client-side GraphQL at GitLab](https://www.youtube.com/watch?v=mCKRJxvMnf0) (video) Illya Klymov and Natalia Tepluhina +- [🎬 From Vuex to Apollo](https://www.youtube.com/watch?v=9knwu87IfU8) (video) by Natalia Tepluhina + - A useful overview of when Apollo might be a better choice than Vuex, and how one could go about the transition +- [🛠 Vuex -> Apollo Migration: a proof-of-concept project](https://gitlab.com/ntepluhina/vuex-to-apollo/blob/master/README.md) + - A collection of examples that show the possible approaches for state management with Vue+GraphQL+(Vuex or Apollo) apps + +### Libraries + +We use [Apollo](https://www.apollographql.com/) (specifically [Apollo Client](https://www.apollographql.com/docs/react/)) and [Vue Apollo](https://github.com/vuejs/vue-apollo) +when using GraphQL for frontend development. + +If you are using GraphQL within a Vue application, the [Usage in Vue](#usage-in-vue) section +can help you learn how to integrate Vue Apollo. + +For other use cases, check out the [Usage outside of Vue](#usage-outside-of-vue) section. + +### Tooling + +- [Apollo Client Devtools](https://github.com/apollographql/apollo-client-devtools) + +#### [Apollo GraphQL VS Code extension](https://marketplace.visualstudio.com/items?itemName=apollographql.vscode-apollo) + +If you use VS Code, the Apollo GraphQL extension supports autocompletion in `.graphql` files. To set up +the GraphQL extension, follow these steps: + +1. Add an `apollo.config.js` file to the root of your `gitlab` local directory. +1. Populate the file with the following content: + + ```javascript + module.exports = { + client: { + includes: ['./app/assets/javascripts/**/*.graphql', './ee/app/assets/javascripts/**/*.graphql'], + service: { + name: 'GitLab', + localSchemaFile: './doc/api/graphql/reference/gitlab_schema.graphql', + }, + }, + }; + ``` + +1. Restart VS Code. + +### Exploring the GraphQL API + Our GraphQL API can be explored via GraphiQL at your instance's -`/-/graphql-explorer` or at [GitLab.com](https://gitlab.com/-/graphql-explorer). +`/-/graphql-explorer` or at [GitLab.com](https://gitlab.com/-/graphql-explorer). Consult the +[GitLab GraphQL API Reference documentation](../../api/graphql/reference) +where needed. You can check all existing queries and mutations on the right side of GraphiQL in its **Documentation explorer**. It's also possible to @@ -10,9 +72,6 @@ their execution by clicking **Execute query** button on the top left:  -We use [Apollo](https://www.apollographql.com/) and [Vue Apollo](https://github.com/vuejs/vue-apollo) for working with GraphQL -on the frontend. - ## Apollo Client To save duplicated clients getting created in different apps, we have a @@ -30,7 +89,7 @@ Default client accepts two parameters: `resolvers` and `config`. ## GraphQL Queries To save query compilation at runtime, webpack can directly import `.graphql` -files. This allows webpack to preprocess the query at compile time instead +files. This allows webpack to pre-process the query at compile time instead of the client doing compilation of queries. To distinguish queries from mutations and fragments, the following naming convention is recommended: @@ -41,7 +100,7 @@ To distinguish queries from mutations and fragments, the following naming conven ### Fragments -Fragments are a way to make your complex GraphQL queries more readable and re-usable. Here is an example of GraphQL fragment: +[Fragments](https://graphql.org/learn/queries/#fragments) are a way to make your complex GraphQL queries more readable and re-usable. Here is an example of GraphQL fragment: ```javascript fragment DesignListItem on Design { @@ -94,7 +153,7 @@ new Vue({ }); ``` -Read more about [Vue Apollo](https://github.com/vuejs/vue-apollo) in the [Vue Apollo documentation](https://vue-apollo.netlify.com/guide/). +Read more about [Vue Apollo](https://github.com/vuejs/vue-apollo) in the [Vue Apollo documentation](https://vue-apollo.netlify.app/guide/). ### Local state with Apollo @@ -206,11 +265,11 @@ const defaultClient = createDefaultClient( Now every single time on attempt to fetch a version, our client will fetch `id` and `sha` from the remote API endpoint and will assign our hardcoded values to `author` and `createdAt` version properties. With this data, frontend developers are able to work on UI part without being blocked by backend. When actual response is added to the API, a custom local resolver can be removed fast and the only change to query/fragment is `@client` directive removal. -Read more about local state management with Apollo in the [Vue Apollo documentation](https://vue-apollo.netlify.com/guide/local-state.html#local-state). +Read more about local state management with Apollo in the [Vue Apollo documentation](https://vue-apollo.netlify.app/guide/local-state.html#local-state). ### Using with Vuex -When Apollo Client is used within Vuex and fetched data is stored in the Vuex store, there is no need in keeping Apollo Client cache enabled. Otherwise we would have data from the API stored in two places - Vuex store and Apollo Client cache. More to say, with Apollo default settings, a subsequent fetch from the GraphQL API could result in fetching data from Apollo cache (in the case where we have the same query and variables). To prevent this behavior, we need to disable Apollo Client cache passing a valid `fetchPolicy` option to its constructor: +When Apollo Client is used within Vuex and fetched data is stored in the Vuex store, there is no need in keeping Apollo Client cache enabled. Otherwise we would have data from the API stored in two places - Vuex store and Apollo Client cache. More to say, with Apollo's default settings, a subsequent fetch from the GraphQL API could result in fetching data from Apollo cache (in the case where we have the same query and variables). To prevent this behavior, we need to disable Apollo Client cache passing a valid `fetchPolicy` option to its constructor: ```javascript import fetchPolicies from '~/graphql_shared/fetch_policy_constants'; @@ -411,18 +470,6 @@ fetchNextPage() { Please note we don't have to save `pageInfo` one more time; `fetchMore` triggers a query `result` hook as well. -#### Limitations - -Currently, bidirectional pagination doesn't work: - -- `hasNextPage` returns a correct value only when we paginate forward using `endCursor` - and `first` parameters. -- `hasPreviousPage` returns a correct value only when we paginate backward using - `startCursor` and `last` parameters. - -This should be resolved in the scope of the issue -[Bi-directional Pagination in GraphQL doesn't work as expected](https://gitlab.com/gitlab-org/gitlab/-/issues/208301). - ### Testing #### Mocking response as component data @@ -599,4 +646,20 @@ defaultClient.query({ query }) .then(result => console.log(result)); ``` -Read more about the [Apollo](https://www.apollographql.com/) client in the [Apollo documentation](https://www.apollographql.com/docs/tutorial/client/). +When [using Vuex](#Using-with-Vuex), disable the cache when: + +- The data is being cached elsewhere +- The use case does not need caching +if the data is being cached elsewhere, or if there is simply no need for it for the given use case. + +```javascript +import createDefaultClient from '~/lib/graphql'; +import fetchPolicies from '~/graphql_shared/fetch_policy_constants'; + +const defaultClient = createDefaultClient( + {}, + { + fetchPolicy: fetchPolicies.NO_CACHE, + }, +); +``` diff --git a/doc/development/fe_guide/icons.md b/doc/development/fe_guide/icons.md index 4fb738f5466..131324e6479 100644 --- a/doc/development/fe_guide/icons.md +++ b/doc/development/fe_guide/icons.md @@ -24,7 +24,7 @@ sprite_icon(icon_name, size: nil, css_class: '') - **icon_name** Use the icon_name that you can find in the SVG Sprite ([Overview is available here](https://gitlab-org.gitlab.io/gitlab-svgs)). - **size (optional)** Use one of the following sizes : 16, 24, 32, 48, 72 (this will be translated into a `s16` class) -- **css_class (optional)** If you want to add additional css classes +- **css_class (optional)** If you want to add additional CSS classes **Example** @@ -67,8 +67,8 @@ export default { - **name** Name of the Icon in the SVG Sprite ([Overview is available here](https://gitlab-org.gitlab.io/gitlab-svgs)). - **size (optional)** Number value for the size which is then mapped to a specific CSS class - (Available Sizes: 8, 12, 16, 18, 24, 32, 48, 72 are mapped to `sXX` css classes) -- **css-classes (optional)** Additional CSS Classes to add to the svg tag. + (Available Sizes: 8, 12, 16, 18, 24, 32, 48, 72 are mapped to `sXX` CSS classes) +- **css-classes (optional)** Additional CSS Classes to add to the SVG tag. ### Usage in HTML/JS @@ -91,7 +91,7 @@ Please use the class `svg-content` around it to ensure nice rendering. ### Usage in Vue -To use an SVG illustrations in a template provide the path as a property and display it through a standard img tag. +To use an SVG illustrations in a template provide the path as a property and display it through a standard `img` tag. Component: diff --git a/doc/development/fe_guide/performance.md b/doc/development/fe_guide/performance.md index aaa6bb16fab..5d2b699c40d 100644 --- a/doc/development/fe_guide/performance.md +++ b/doc/development/fe_guide/performance.md @@ -65,7 +65,7 @@ within the `pages` directory correspond to Rails controllers and actions. These auto-generated bundles will be automatically included on the corresponding pages. -For example, if you were to visit <https://gitlab.com/gitlab-org/gitlab/issues>, +For example, if you were to visit <https://gitlab.com/gitlab-org/gitlab/-/issues>, you would be accessing the `app/controllers/projects/issues_controller.rb` controller with the `index` action. If a corresponding file exists at `pages/projects/issues/index/index.js`, it will be compiled into a webpack diff --git a/doc/development/fe_guide/style/vue.md b/doc/development/fe_guide/style/vue.md index 46305cc7217..1a64db443bc 100644 --- a/doc/development/fe_guide/style/vue.md +++ b/doc/development/fe_guide/style/vue.md @@ -53,7 +53,7 @@ Please check this [rules](https://github.com/vuejs/eslint-plugin-vue#bulb-rules) ## Naming -1. **Extensions**: Use `.vue` extension for Vue components. Do not use `.js` as file extension ([#34371](https://gitlab.com/gitlab-org/gitlab-foss/issues/34371)). +1. **Extensions**: Use `.vue` extension for Vue components. Do not use `.js` as file extension ([#34371](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34371)). 1. **Reference Naming**: Use PascalCase for their instances: ```javascript diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index 972c2ded9c9..0d77e4d129b 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -6,9 +6,9 @@ To get started with Vue, read through [their documentation](https://vuejs.org/v2 What is described in the following sections can be found in these examples: -- web ide: <https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/ide/stores> -- security products: <https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/assets/javascripts/vue_shared/security_reports> -- registry: <https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/registry/stores> +- [Web IDE](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/ide/stores) +- [Security products](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/assets/javascripts/vue_shared/security_reports) +- [Registry](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/registry/stores) ## Vue architecture @@ -16,7 +16,7 @@ All new features built with Vue.js must follow a [Flux architecture](https://fac The main goal we are trying to achieve is to have only one data flow and only one data entry. In order to achieve this goal we use [vuex](#vuex). -You can also read about this architecture in vue docs about [state management](https://vuejs.org/v2/guide/state-management.html#Simple-State-Management-from-Scratch) +You can also read about this architecture in Vue docs about [state management](https://vuejs.org/v2/guide/state-management.html#Simple-State-Management-from-Scratch) and about [one way data flow](https://vuejs.org/v2/guide/components.html#One-Way-Data-Flow). ### Components and Store @@ -59,7 +59,7 @@ To do that, provide the data through `data` attributes in the HTML element and q _Note:_ You should only do this while initializing the application, because the mounted element will be replaced with Vue-generated DOM. The advantage of providing data from the DOM to the Vue instance through `props` in the `render` function -instead of querying the DOM inside the main vue component is that makes tests easier by avoiding the need to +instead of querying the DOM inside the main Vue component is that makes tests easier by avoiding the need to create a fixture or an HTML element in the unit test. See the following example: ```javascript diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index 309aecd7978..d4d1ec74591 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -96,7 +96,7 @@ As a guideline: Before toggling any feature flag, check that there are no ongoing significant incidents on GitLab.com. You can do this by checking the `#production` and `#incident-management` Slack channels, or looking for -[open incident issues](https://gitlab.com/gitlab-com/gl-infra/production/issues/?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=incident) +[open incident issues](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=incident) (although check the dates and times). We do not want to introduce changes during an incident, as it can make @@ -113,17 +113,58 @@ When you begin to enable the feature, please link to the relevant Feature Flag Rollout Issue within a Slack thread of the first `/chatops` command you make so people can understand the change if they need to. -To enable a feature for 25% of all users, run the following in Slack: +To enable a feature for 25% of the time, run the following in Slack: ```shell /chatops run feature set new_navigation_bar 25 ``` +This sets a feature flag to `true` based on the following formula: + +```ruby +feature_flag_state = rand < (25 / 100.0) +``` + This will enable the feature for GitLab.com, with `new_navigation_bar` being the name of the feature. This command does *not* enable the feature for 25% of the total users. Instead, when the feature is checked with `enabled?`, it will return `true` 25% of the time. +To enable a feature for 25% of actors such as users, projects, or groups, +run the following in Slack: + +```shell +/chatops run feature set some_feature 25 --actors +``` + +This sets a feature flag to `true` based on the following formula: + +```ruby +feature_flag_state = Zlib.crc32("some_feature<Actor>:#{actor.id}") % (100 * 1_000) < 25 * 1_000] +# where <Actor>: is a `User`, `Group`, `Project` and actor is an instance +``` + +During development, based on the nature of the feature, an actor choice +should be made. + +For user focused features: + +```ruby +Feature.enabled?(:feature_cool_avatars, current_user) +``` + +For group or namespace level features: + +```ruby +Feature.enabled?(:feature_cooler_groups, group) +``` + +For project level features: + +```ruby +Feature.enabled?(:feature_ice_cold_projects, project) +``` + If you are not certain what percentages to use, simply use the following steps: 1. 25% @@ -158,15 +199,21 @@ you run these 2 commands: ```shell /chatops run feature set --project=gitlab-org/gitlab some_feature true -/chatops run feature set some_feature 25 +/chatops run feature set some_feature 25 --actors +``` + +Then `some_feature` will be enabled for both 25% of actors and always when interacting with +`gitlab-org/gitlab`. This is a good idea if the feature flag development makes use of group +actors. + +```ruby +Feature.enabled?(:some_feature, group) ``` -Then `some_feature` will be enabled for both 25% of users and all users interacting with -`gitlab-org/gitlab`. +NOTE: -NOTE: **Note:** **Percentage of time** rollout is not a good idea if what you want is to make sure a feature -is always on or off to the users. +is always on or off to the users. In that case, **Percentage of actors** rollout is a better method. ### Feature flag change logging @@ -174,7 +221,7 @@ Any feature flag change that affects GitLab.com (production) will automatically be logged in an issue. The issue is created in the -[gl-infra/feature-flag-log](https://gitlab.com/gitlab-com/gl-infra/feature-flag-log/issues?scope=all&utf8=%E2%9C%93&state=closed) +[gl-infra/feature-flag-log](https://gitlab.com/gitlab-com/gl-infra/feature-flag-log/-/issues?scope=all&utf8=%E2%9C%93&state=closed) project, and it will at minimum log the Slack handle of person enabling a feature flag, the time, and the name of the flag being changed. diff --git a/doc/development/feature_flags/development.md b/doc/development/feature_flags/development.md index b20db295919..a44bc70439e 100644 --- a/doc/development/feature_flags/development.md +++ b/doc/development/feature_flags/development.md @@ -23,6 +23,9 @@ class Foo < ActiveRecord::Base end ``` +Only models that `include FeatureGate` or expose `flipper_id` method can be +used as an actor for `Feature.enabled?`. + Features that are developed and are intended to be merged behind a feature flag should not include a changelog entry. The entry should either be added in the merge request removing the feature flag or the merge request where the default value of @@ -87,6 +90,10 @@ this method you can expose the state of a feature flag as follows: ```ruby before_action do + # Prefer to scope it per project or user e.g. + push_frontend_feature_flag(:vim_bindings, project) + + # Avoid, if possible push_frontend_feature_flag(:vim_bindings) end @@ -115,17 +122,30 @@ how to access feature flags in a Vue component. ### Specs -In the test environment `Feature.enabled?` is stubbed to always respond to `true`, -so we make sure behavior under feature flag doesn't go untested in some non-specific -contexts. +Our Flipper engine in the test environment works in a memory mode `Flipper::Adapters::Memory`. +`production` and `development` modes use `Flipper::Adapters::ActiveRecord`. + +### `stub_feature_flags: true` (default and preferred) -Whenever a feature flag is present, make sure to test _both_ states of the -feature flag. +In this mode Flipper is configured to use `Flipper::Adapters::Memory` and mark all feature +flags to be on-by-default and persisted on a first use. This overwrites the `default_enabled:` +of `Feature.enabled?` and `Feature.disabled?` returning always `true` unless feature flag +is persisted. + +Make sure behavior under feature flag doesn't go untested in some non-specific contexts. See the [testing guide](../testing_guide/best_practices.md#feature-flags-in-tests) for information and examples on how to stub feature flags in tests. +### `stub_feature_flags: false` + +This disables a memory-stubbed flipper, and uses `Flipper::Adapters::ActiveRecord` +a mode that is used by `production` and `development`. + +You should use this mode only when you really want to tests aspects of Flipper +with how it interacts with `ActiveRecord`. + ### Enabling a feature flag (in development) In the rails console (`rails c`), enter the following command to enable your feature flag @@ -139,3 +159,9 @@ Similarly, the following command will disable a feature flag: ```ruby Feature.disable(:feature_flag_name) ``` + +You can as well enable feature flag for a given gate: + +```ruby +Feature.enable(:feature_flag_name, Project.find_by_full_path("root/my-project")) +``` diff --git a/doc/development/filtering_by_label.md b/doc/development/filtering_by_label.md index 19dece0d5c9..ef92bd35985 100644 --- a/doc/development/filtering_by_label.md +++ b/doc/development/filtering_by_label.md @@ -40,13 +40,13 @@ In particular, note that: This is more complicated than is ideal. It makes the query construction more prone to errors (such as -[issue #15557](https://gitlab.com/gitlab-org/gitlab-foss/issues/15557)). +[issue #15557](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/15557)). ## Attempt A: WHERE EXISTS ### Attempt A1: use multiple subqueries with WHERE EXISTS -In [issue #37137](https://gitlab.com/gitlab-org/gitlab-foss/issues/37137) +In [issue #37137](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/37137) and its associated [merge request](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14022), we tried to replace the `GROUP BY` with multiple uses of `WHERE EXISTS`. For the example above, this would give: @@ -83,7 +83,7 @@ Having [removed MySQL support in GitLab 12.1](https://about.gitlab.com/blog/2019 using [PostgreSQL's arrays](https://www.postgresql.org/docs/11/arrays.html) became more tractable as we didn't have to support two databases. We discussed denormalizing the `label_links` table for querying in -[issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/issues/49651), +[issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49651), with two options: label IDs and titles. We can think of both of those as array columns on `issues`, `merge_requests`, @@ -147,7 +147,7 @@ WHERE label_titles @> ARRAY['Plan', 'backend'] ``` -And our [tests in issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/issues/49651#note_188777346) +And our [tests in issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49651#note_188777346) showed that this could be fast. However, at present, the disadvantages outweigh the advantages. diff --git a/doc/development/geo.md b/doc/development/geo.md index bf56340f8ec..5c781a60bac 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -94,7 +94,7 @@ projects that need updating. Those projects can be: [Geo admin panel](../user/admin_area/geo_nodes.md). When we fail to fetch a repository on the secondary `RETRIES_BEFORE_REDOWNLOAD` -times, Geo does a so-called _redownload_. It will do a clean clone +times, Geo does a so-called _re-download_. It will do a clean clone into the `@geo-temporary` directory in the root of the storage. When it's successful, we replace the main repo with the newly cloned one. @@ -218,7 +218,7 @@ the performance of many synchronization operations. FDW is a PostgreSQL extension ([`postgres_fdw`](https://www.postgresql.org/docs/11/postgres-fdw.html)) that is enabled within the Geo Tracking Database (on a **secondary** node), which allows it -to connect to the readonly database replica and perform queries and filter +to connect to the read-only database replica and perform queries and filter data from both instances. This persistent connection is configured as an FDW server @@ -226,7 +226,7 @@ named `gitlab_secondary`. This configuration exists within the database's user context only. To access the `gitlab_secondary`, GitLab needs to use the same database user that had previously been configured. -The Geo Tracking Database accesses the readonly database replica via FDW as a regular user, +The Geo Tracking Database accesses the read-only database replica via FDW as a regular user, limited by its own restrictions. The credentials are configured as a `USER MAPPING` associated with the `SERVER` mapped previously (`gitlab_secondary`). diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index d72f7cc4cc1..85fecc41cfb 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -257,6 +257,8 @@ For example, to add support for files referenced by a `Widget` model with a class Geo::WidgetRegistry < Geo::BaseRegistry include Geo::StateMachineRegistry + MODEL_FOREIGN_KEY = :widget_id + belongs_to :widget, class_name: 'Widget' end ``` @@ -305,6 +307,8 @@ For example, to add support for files referenced by a `Widget` model with a specify 'factory is valid' do expect(registry).to be_valid end + + include_examples 'a Geo framework registry' end ``` @@ -356,7 +360,8 @@ Widgets should now be replicated by Geo! end ``` -1. Add fields `widget_count`, `widget_checksummed_count`, and `widget_checksum_failed_count` +1. Add fields `widget_count`, `widget_checksummed_count`, `widget_checksum_failed_count`, + `widget_synced_count` and `widget_failed_count` to `GeoNodeStatus#RESOURCE_STATUS_FIELDS` array in `ee/app/models/geo_node_status.rb`. 1. Add the same fields to `GeoNodeStatus#PROMETHEUS_METRICS` hash in `ee/app/models/geo_node_status.rb`. @@ -370,6 +375,8 @@ Widgets should now be replicated by Geo! self.widget_count = Geo::WidgetReplicator.model.count self.widget_checksummed_count = Geo::WidgetReplicator.checksummed.count self.widget_checksum_failed_count = Geo::WidgetReplicator.checksum_failed.count + self.widget_synced_count = Geo::WidgetReplicator.synced_count + self.widget_failed_count = Geo::WidgetReplicator.failed_count ``` 1. Make sure `Widget` model has `checksummed` and `checksum_failed` scopes. @@ -450,7 +457,7 @@ Widgets should now be verified by Geo! end ``` -1. Create `ee/app/graphql/types/geo/package_file_registry_type.rb`: +1. Create `ee/app/graphql/types/geo/widget_registry_type.rb`: ```ruby # frozen_string_literal: true diff --git a/doc/development/git_object_deduplication.md b/doc/development/git_object_deduplication.md index 938882ba5a2..06af34d9232 100644 --- a/doc/development/git_object_deduplication.md +++ b/doc/development/git_object_deduplication.md @@ -89,7 +89,7 @@ are as follows: (`pool.source_project`) > TODO Fix invalid SQL data for pools created prior to GitLab 11.11 -> <https://gitlab.com/gitlab-org/gitaly/issues/1653>. +> <https://gitlab.com/gitlab-org/gitaly/-/issues/1653>. ### Assumptions @@ -133,7 +133,7 @@ are as follows: repository. > TODO should forks of forks be deduplicated? -> <https://gitlab.com/gitlab-org/gitaly/issues/1532> +> <https://gitlab.com/gitlab-org/gitaly/-/issues/1532> ### Consequences @@ -191,4 +191,4 @@ the secondary, at which stage Git objects will get deduplicated. > TODO How do we handle the edge case where at the time the Geo > secondary tries to create the pool repository, the source project does -> not exist? <https://gitlab.com/gitlab-org/gitaly/issues/1533> +> not exist? <https://gitlab.com/gitlab-org/gitaly/-/issues/1533> diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 5e5cae7228b..417c96a44dd 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -54,7 +54,7 @@ The process for adding new Gitaly features is: These steps often overlap. It is possible to use an unreleased version of Gitaly and `gitaly-proto` during testing and development. -- See the [Gitaly repo](https://gitlab.com/gitlab-org/gitaly/blob/master/CONTRIBUTING.md#development-and-testing-with-a-custom-gitaly-proto) for instructions on writing server side code with an unreleased protocol. +- See the [Gitaly repository](https://gitlab.com/gitlab-org/gitaly/blob/master/CONTRIBUTING.md#development-and-testing-with-a-custom-gitaly-proto) for instructions on writing server side code with an unreleased protocol. - See [below](#running-tests-with-a-locally-modified-version-of-gitaly) for instructions on running GitLab CE tests with a modified version of Gitaly. - In GDK run `gdk install` and restart `gdk run` (or `gdk run app`) to use a locally modified Gitaly version for development @@ -67,7 +67,7 @@ This should make it easier to contribute for developers who are less comfortable writing Go code. There is documentation for this approach in [the Gitaly -repo](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/ruby_endpoint.md). +repository](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/ruby_endpoint.md). ## Gitaly-Related Test Failures @@ -85,7 +85,7 @@ While Gitaly can handle all Git access, many of GitLab customers still run Gitaly atop NFS. The legacy Rugged implementation for Git calls may be faster than the Gitaly RPC due to N+1 Gitaly calls and other reasons. See [the -issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/57317) for more +issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57317) for more details. Until GitLab has eliminated most of these inefficiencies or the use of @@ -323,8 +323,8 @@ the integration by using GDK: 1. Navigate to GDK's root directory. 1. Make sure you have the proper branch checked out for Gitaly. 1. Recompile it with `make gitaly-setup` and restart the service with `gdk restart gitaly`. - 1. Make sure your setup is runnig: `gdk status | grep praefect`. - 1. Check what config file is used: `cat ./services/praefect/run | grep praefect` value of the `-config` flag + 1. Make sure your setup is running: `gdk status | grep praefect`. + 1. Check what configuration file is used: `cat ./services/praefect/run | grep praefect` value of the `-config` flag 1. Uncomment `prometheus_listen_addr` in the configuration file and run `gdk restart gitaly`. 1. Make sure that the flag is not enabled yet: diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index 5d37d2f119f..5a490737f37 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -121,12 +121,12 @@ also reduce pressure on the system as a whole. ## Refreshing import JIDs -GitLab includes a worker called `StuckImportJobsWorker` that will periodically -run and mark project imports as failed if they have been running for more than -15 hours. For GitHub projects, this poses a bit of a problem: importing large -projects could take several hours depending on how often we hit the GitHub rate -limit (more on this below), but we don't want `StuckImportJobsWorker` to mark -our import as failed because of this. +GitLab includes a worker called `Gitlab::Import::StuckProjectImportJobsWorker` +that will periodically run and mark project imports as failed if they have been +running for more than 15 hours. For GitHub projects, this poses a bit of a +problem: importing large projects could take several hours depending on how +often we hit the GitHub rate limit (more on this below), but we don't want +`Gitlab::Import::StuckProjectImportJobsWorker` to mark our import as failed because of this. To prevent this from happening we periodically refresh the expiration time of the import process. This works by storing the JID of the import job in the diff --git a/doc/development/go_guide/dependencies.md b/doc/development/go_guide/dependencies.md new file mode 100644 index 00000000000..b85344635c6 --- /dev/null +++ b/doc/development/go_guide/dependencies.md @@ -0,0 +1,175 @@ +# Dependency Management in Go + +Go takes an unusual approach to dependency management, in that it is +source-based instead of artifact-based. In an artifact-based dependency +management system, packages consist of artifacts generated from source code and +are stored in a separate repository system from source code. For example, many +NodeJS packages use `npmjs.org` as a package repository and `github.com` as a +source repository. On the other hand, packages in Go *are* source code and +releasing a package does not involve artifact generation or a separate +repository. Go packages must be stored in a version control repository on a VCS +server. Dependencies are fetched directly from their VCS server or via an +intermediary proxy which itself fetches them from their VCS server. + +## Versioning + +Go 1.11 introduced modules and first-class package versioning to the Go ecosystem. +Prior to this, Go did not have any well-defined mechanism for version management. +While 3rd party version management tools existed, the default Go experience had +no support for versioning. + +Go modules use [semantic versioning](https://semver.org). The versions of a +module are defined as VCS (version control system) tags that are valid semantic +versions prefixed with `v`. For example, to release version `1.0.0` of +`gitlab.com/my/project`, the developer must create the Git tag `v1.0.0`. + +For major versions other than 0 and 1, the module name must be suffixed with +`/vX` where X is the major version. For example, version `v2.0.0` of +`gitlab.com/my/project` must be named and imported as +`gitlab.com/my/project/v2`. + +Go uses 'pseudo-versions', which are special semantic versions that reference a +specific VCS commit. The prerelease component of the semantic version must be or +end with a timestamp and the first 12 characters of the commit identifier: + +- `vX.0.0-yyyymmddhhmmss-abcdefabcdef`, when no earlier tagged commit exists for X. +- `vX.Y.Z-pre.0.yyyymmddhhmmss-abcdefabcdef`, when most recent prior tag is vX.Y.Z-pre. +- `vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdefabcdef`, when most recent prior tag is vX.Y.Z. + +If a VCS tag matches one of these patterns, it is ignored. + +For a complete understanding of Go modules and versioning, see [this series of +blog posts](https://blog.golang.org/using-go-modules) on the official Go +website. + +## 'Module' vs 'Package' + +- A package is a folder containing `*.go` files. +- A module is a folder containing a `go.mod` file. +- A module is *usually* also a package, that is a folder containing a `go.mod` + file and `*.go` files. +- A module may have subdirectories, which may be packages. +- Modules usually come in the form of a VCS repository (Git, SVN, Hg, and so on). +- Any subdirectories of a module that themselves are modules are distinct, + separate modules and are excluded from the containing module. + - Given a module `repo`, if `repo/sub` contains a `go.mod` file then + `repo/sub` and any files contained therein are a separate module and not a + part of `repo`. + +## Naming + +The name of a module or package, excluding the standard library, must be of the +form `(sub.)*domain.tld(/path)*`. This is similar to a URL, but is not a URL. +The package name does not have a scheme (such as `https://`) and cannot have a +port number. `example.com:8443/my/package` is not a valid name. + +## Fetching Packages + +Prior to Go 1.12, the process for fetching a package was as follows: + +1. Query `https://{package name}?go-get=1`. +1. Scan the response for the `go-import` meta tag. +1. Fetch the repository indicated by the meta tag using the indicated VCS. + +The meta tag should have the form `<meta name="go-import" content="{prefix} +{vcs} {url}">`. For example, `gitlab.com/my/project git +https://gitlab.com/my/project.git` indicates that packages beginning with +`gitlab.com/my/project` should be fetched from +`https://gitlab.com/my/project.git` using Git. + +## Fetching Modules + +Go 1.12 introduced checksum databases and module proxies. + +### Checksums + +In addition to `go.mod`, a module will have a `go.sum` file. This file records a +SHA-256 checksum of the code and the `go.mod` file of every version of every +dependency that is referenced by the module or one of the module's dependencies. +Go continually updates `go.sum` as new dependencies are referenced. + +When Go fetches the dependencies of a module, if those dependencies already have +an entry in `go.sum`, Go will verify the checksum of these dependencies. If the +checksum does not match what is in `go.sum`, the build will fail. This ensures +that a given version of a module cannot be changed by its developers or by a +malicious party without causing build failures. + +Go 1.12+ can be configured to use a checksum database. If configured to do so, +when Go fetches a dependency and there is no corresponding entry in `go.sum`, Go +will query the configured checksum database(s) for the checksum of the +dependency instead of calculating it from the downloaded dependency. If the +dependency cannot be found in the checksum database, the build will fail. If the +downloaded dependency's checksum does not match the result from the checksum +database, the build will fail. The following environment variables control this: + +- `GOSUMDB` identifies the name, and optionally the public key and server URL, + of the checksum database to query. + - A value of `off` will entirely disable checksum database queries. + - Go 1.13+ uses `sum.golang.org` if `GOSUMDB` is not defined. +- `GONOSUMDB` is a comma-separated list of module suffixes that checksum + database queries should be disabled for. Wildcards are supported. +- `GOPRIVATE` is a comma-separated list of module names that has the same + function as `GONOSUMDB` in addition to disabling other features. + +### Proxies + +Go 1.12+ can be configured to fetch modules from a Go proxy instead of directly +from the module's VCS. If configured to do so, when Go fetches a dependency, it +attempts to fetch the dependency from the configured proxies, in order. The +following environment variables control this: + +- `GOPROXY` is a comma-separated list of module proxies to query. + - A value of `direct` will entirely disable module proxy queries. + - If the last entry in the list is `direct`, Go will fall back to the process + described [above](#fetching-packages) if none of the proxies can provide the + dependency. + - Go 1.13+ uses `proxy.golang.org,direct` if `GOPROXY` is not defined. +- `GONOPROXY` is a comma-separated list of module suffixes that should be + fetched directly and not from a proxy. Wildcards are supported. +- `GOPRIVATE` is a comma-separated list of module names that has the same + function as `GONOPROXY` in addition to disabling other features. + +### Fetching + +From Go 1.12 onward, the process for fetching a module or package is as follows: + +1. If `GOPROXY` is a list of proxies and the module is not excluded by + `GONOPROXY` or `GOPRIVATE`, query them in order, and stop at the first valid + response. +1. If `GOPROXY` is `direct`, or the module is excluded, or `GOPROXY` ends with + `,direct` and no proxy provided the module, fall back. + 1. Query `https://{module or package name}?go-get=1`. + 1. Scan the response for the `go-import` meta tag. + 1. Fetch the repository indicated by the meta tag using the indicated VCS. + 1. If the `{vcs}` field is `mod`, the URL should be treated as a module proxy instead of a VCS. +1. If the module is being fetched directly and not as a dependency, stop. +1. If `go.sum` contains an entry corresponding to the module, validate the checksum and stop. +1. If `GOSUMDB` identifies a checksum database and the module is not excluded by + `GONOSUMDB` or `GOPRIVATE`, retrieve the module's checksum, add it to + `go.sum`, and validate the downloaded source against it. +1. If `GOSUMDB` is `off` or the module is excluded, calculate a checksum from + the downloaded source and add it to `go.sum`. + +The downloaded source must contain a `go.mod` file. The `go.mod` file must +contain a `module` directive that specifies the name of the module. If the +module name as specified by `go.mod` does not match the name that was used to +fetch the module, the module will fail to compile. + +If the module is being fetched directly and no version was specified, or if the +module is being added as a dependency and no version was specified, Go uses the +most recent version of the module. If the module is fetched from a proxy, Go +queries the proxy for a list of versions and chooses the latest. If the module is +fetched directly, Go queries the repository for a list of tags and chooses the +latest that is also a valid semantic version. + +## Authenticating + +In versions prior to Go 1.13, support for authenticating requests made by Go was +somewhat inconsistent. Go 1.13 improved support for `.netrc` authentication. If +a request is made over HTTPS and a matching `.netrc` entry can be found, Go will +add HTTP Basic authentication credentials to the request. Go will not +authenticate requests made over HTTP. Go will reject HTTP-only entries in +`GOPROXY` that have embedded credentials. + +In a future version, Go may add support for arbitrary authentication headers. +Follow [golang/go#26232](https://github.com/golang/go/issues/26232) for details. diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index fe69a4205f8..5a5e163e142 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -17,6 +17,22 @@ experiences. Several projects were started with different standards and they can still have specifics. They will be described in their respective `README.md` or `PROCESS.md` files. +## Dependency Management + +Go uses a source-based strategy for dependency management. Dependencies are +downloaded as source from their source repository. This differs from the more +common artifact-based strategy where dependencies are downloaded as artifacts +from a package repository that is separate from the dependency's source +repository. + +Go did not have first-class support for version management prior to 1.11. That +version introduced Go modules and the use of semantic versioning. Go 1.12 +introduced module proxies, which can serve as an intermediate between clients +and source version control systems, and checksum databases, which can be used to +verify the integrity of dependency downloads. + +See [Dependency Management in Go](dependencies.md) for more details. + ## Code Review We follow the common principles of @@ -105,7 +121,7 @@ Including a `.golangci.yml` in the root directory of the project allows for configuration of `golangci-lint`. All options for `golangci-lint` are listed in this [example](https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml). -Once [recursive includes](https://gitlab.com/gitlab-org/gitlab-foss/issues/56836) +Once [recursive includes](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/56836) become available, you will be able to share job templates like this [analyzer](https://gitlab.com/gitlab-org/security-products/ci-templates/raw/master/includes-dev/analyzer.yml). @@ -260,7 +276,7 @@ easier to debug. For example: -```go +```golang // Wrap the error return nil, fmt.Errorf("get cache %s: %w", f.Name, err) @@ -390,7 +406,7 @@ builds](https://docs.docker.com/develop/develop-images/multistage-build/): dependencies. - They generate a small, self-contained image, derived from `Scratch`. -Generated docker images should have the program at their `Entrypoint` to create +Generated Docker images should have the program at their `Entrypoint` to create portable commands. That way, anyone can run the image, and without parameters it will display its help message (if `cli` has been used). diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index 8f077e613b7..7e08da8162b 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -155,7 +155,7 @@ refresh_service.execute(oldrev, newrev, ref) See ["Why is it bad style to `rescue Exception => e` in Ruby?"](https://stackoverflow.com/questions/10048173/why-is-it-bad-style-to-rescue-exception-e-in-ruby). _**Note:** This rule is [enforced automatically by -Rubocop](https://gitlab.com/gitlab-org/gitlab-foss/blob/8-4-stable/.rubocop.yml#L911-914)._ +RuboCop](https://gitlab.com/gitlab-org/gitlab-foss/blob/8-4-stable/.rubocop.yml#L911-914)._ ## Do not use inline JavaScript in views diff --git a/doc/development/hash_indexes.md b/doc/development/hash_indexes.md index bc962ac0cd6..1ed76e35f69 100644 --- a/doc/development/hash_indexes.md +++ b/doc/development/hash_indexes.md @@ -1,6 +1,6 @@ # Hash Indexes -PostgreSQL supports hash indexes besides the regular btree +PostgreSQL supports hash indexes besides the regular B-tree indexes. Hash indexes however are to be avoided at all costs. While they may _sometimes_ provide better performance the cost of rehashing can be very high. More importantly: at least until PostgreSQL 10.0 hash indexes are not @@ -17,4 +17,4 @@ documentation: RuboCop is configured to register an offense when it detects the use of a hash index. -Instead of using hash indexes you should use regular btree indexes. +Instead of using hash indexes you should use regular B-tree indexes. diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index a81e656fc27..bdd372e90ed 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -260,8 +260,22 @@ n_("%{project_name}", "%d projects selected", count) % { project_name: 'GitLab' ### Namespaces -Sometimes you need to add some context to the text that you want to translate -(if the word occurs in a sentence and/or the word is ambiguous). +A namespace is a way to group translations that belong together. They provide context to our translators by adding a prefix followed by the bar symbol (`|`). For example: + +```ruby +'Namespace|Translated string' +``` + +A namespace provide the following benefits: + +- It addresses ambiguity in words, for example: `Promotions|Promote` vs `Epic|Promote` +- It allows translators to focus on translating externalized strings that belong to the same product area rather than arbitrary ones. +- It gives a linguistic context to help the translator. + +In some cases, namespaces don't make sense, for example, +for ubiquitous UI words and phrases such as "Cancel" or phrases like "Save changes" a namespace could +be counterproductive. + Namespaces should be PascalCase. - In Ruby/HAML: @@ -292,7 +306,7 @@ const dateFormat = createDateTimeFormat({ year: 'numeric', month: 'long', day: ' console.log(dateFormat.format(new Date('2063-04-05'))) // April 5, 2063 ``` -This makes use of [`Intl.DateTimeFormat`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DateTimeFormat). +This makes use of [`Intl.DateTimeFormat`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/DateTimeFormat). - In Ruby/HAML, we have two ways of adding format to dates and times: diff --git a/doc/development/i18n/merging_translations.md b/doc/development/i18n/merging_translations.md index 5d9dbd23efa..e0fcaddda4c 100644 --- a/doc/development/i18n/merging_translations.md +++ b/doc/development/i18n/merging_translations.md @@ -20,7 +20,7 @@ doesn't do. Create a new pipeline at `https://gitlab.com/gitlab-org/gitlab/pipel If there are validation errors, the easiest solution is to disapprove the offending string in CrowdIn, leaving a comment with what is required to fix the offense. There is an -[issue](https://gitlab.com/gitlab-org/gitlab/issues/23256) +[issue](https://gitlab.com/gitlab-org/gitlab/-/issues/23256) suggesting to automate this process. Disapproving will exclude the invalid translation, the merge request will be updated within a few minutes. @@ -31,7 +31,7 @@ clicking `Pause sync` on the [CrowdIn integration settings page](https://translate.gitlab.com/project/gitlab-ee/settings#integration). When all failures are resolved, the translations need to be double -checked once more as discussed in [confidential issue](../../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab/issues/19485`. +checked once more as discussed in [confidential issue](../../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab/-/issues/19485`. ## Merging translations @@ -40,7 +40,7 @@ translations can be merged into the master branch. When merging the translations make sure to check the **Remove source branch** checkbox, so CrowdIn recreates the `master-i18n` from master after the new translation was merged. -We are discussing [automating this entire process](https://gitlab.com/gitlab-org/gitlab/issues/19896). +We are discussing [automating this entire process](https://gitlab.com/gitlab-org/gitlab/-/issues/19896). ## Recreate the merge request @@ -53,3 +53,18 @@ and delete the This might be needed when the merge request contains failures that have been fixed on master. + +## Recreate the GitLab integration in CrowdIn + +NOTE: ***Note:** +These instructions work only for GitLab Team Members. + +If for some reason the GitLab integration in CrowdIn does not exist, it can be +recreated by the following steps: + +1. Sign in to GitLab as `gitlab-crowdin-bot` (If you're a GitLab Team Member, find credentials in the GitLab shared [1Password account](https://about.gitlab.com/handbook/security/#1password-for-teams) +1. Sign in to Crowdin with the GitLab integration +1. Navigate to Settings > Integrations > GitLab > Set Up Integration +1. Select `gitlab-org/gitlab` repository +1. On `Select Branches for Translation`, select `master` +1. Ensure the `Service Branch Name` is `master-i18n` diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index 837da349f7e..8c1f62330f4 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -5,12 +5,15 @@ are very appreciative of the work done by translators and proofreaders! ## Proofreaders +<!-- vale gitlab.Spelling = NO --> - Albanian - Proofreaders needed. - Amharic - Tsegaselassie Tadesse - [GitLab](https://gitlab.com/tsega), [CrowdIn](https://crowdin.com/profile/tsegaselassi/activity) - Arabic - Proofreaders needed. +- Bosnian + - Proofreaders needed. - Bulgarian - Lyubomir Vasilev - [CrowdIn](https://crowdin.com/profile/lyubomirv) - Catalan @@ -26,6 +29,8 @@ are very appreciative of the work done by translators and proofreaders! - Chinese Traditional, Hong Kong 繁體中文 (香港) - Victor Wu - [GitLab](https://gitlab.com/victorwuky), [CrowdIn](https://crowdin.com/profile/victorwu) - Ivan Ip - [GitLab](https://gitlab.com/lifehome), [CrowdIn](https://crowdin.com/profile/lifehome) +- Croatian + - Proofreaders needed. - Czech - Jan Urbanec - [GitLab](https://gitlab.com/TatranskyMedved), [CrowdIn](https://crowdin.com/profile/Tatranskymedved) - Danish @@ -33,11 +38,11 @@ are very appreciative of the work done by translators and proofreaders! - Dutch - Emily Hendle - [GitLab](https://gitlab.com/pundachan), [CrowdIn](https://crowdin.com/profile/pandachan) - Esperanto -- Lyubomir Vasilev - [CrowdIn](https://crowdin.com/profile/lyubomirv) + - Lyubomir Vasilev - [CrowdIn](https://crowdin.com/profile/lyubomirv) - Estonian - Proofreaders needed. - Filipino - - Proofreaders needed. + - Andrei Jiroh Halili - [GitLab](https://gitlab.com/AJHalili2006DevPH), [Crowdin](https://crowdin.com/profile/AndreiJirohHaliliDev2006) - French - Davy Defaud - [GitLab](https://gitlab.com/DevDef), [CrowdIn](https://crowdin.com/profile/DevDef) - Galician @@ -50,6 +55,8 @@ are very appreciative of the work done by translators and proofreaders! - Proofreaders needed. - Hebrew - Yaron Shahrabani - [GitLab](https://gitlab.com/yarons), [CrowdIn](https://crowdin.com/profile/YaronSh) +- Hindi + - Proofreaders needed. - Hungarian - Proofreaders needed. - Indonesian @@ -62,6 +69,7 @@ are very appreciative of the work done by translators and proofreaders! - Hiroyuki Sato - [GitLab](https://gitlab.com/hiroponz), [CrowdIn](https://crowdin.com/profile/hiroponz) - Tomo Dote - [GitLab](https://gitlab.com/fu7mu4), [CrowdIn](https://crowdin.com/profile/fu7mu4) - Hiromi Nozawa - [GitLab](https://gitlab.com/hir0mi), [CrowdIn](https://crowdin.com/profile/hir0mi) + - Takuya Noguchi - [GitLab](https://gitlab.com/tnir), [CrowdIn](https://crowdin.com/profile/tnir) - Korean - Chang-Ho Cha - [GitLab](https://gitlab.com/changho-cha), [CrowdIn](https://crowdin.com/profile/zzazang) - Ji Hun Oh - [GitLab](https://gitlab.com/Baw-Appie), [CrowdIn](https://crowdin.com/profile/BawAppie) @@ -74,7 +82,6 @@ are very appreciative of the work done by translators and proofreaders! - Filip Mech - [GitLab](https://gitlab.com/mehenz), [CrowdIn](https://crowdin.com/profile/mehenz) - Maksymilian Roman - [GitLab](https://gitlab.com/villaincandle), [CrowdIn](https://crowdin.com/profile/villaincandle) - Portuguese - - Proofreaders needed. - Diogo Trindade - [GitLab](https://gitlab.com/luisdiogo2071317), [CrowdIn](https://crowdin.com/profile/ldiogotrindade) - Portuguese, Brazilian - Paulo George Gomes Bezerra - [GitLab](https://gitlab.com/paulobezerra), [CrowdIn](https://crowdin.com/profile/paulogomes.rep) @@ -88,21 +95,22 @@ are very appreciative of the work done by translators and proofreaders! - NickVolynkin - [Crowdin](https://crowdin.com/profile/NickVolynkin) - Andrey Komarov - [GitLab](https://gitlab.com/elkamarado), [Crowdin](https://crowdin.com/profile/kamarado) - Iaroslav Postovalov - [GitLab](https://gitlab.com/CMDR_Tvis), [Crowdin](https://crowdin.com/profile/CMDR_Tvis) -- Serbian (Cyrillic) - - Proofreaders needed. -- Serbian (Latin) +- Serbian (Latin and Cyrillic) - Proofreaders needed. - Slovak - Proofreaders needed. - Spanish - Pedro Garcia - [GitLab](https://gitlab.com/pedgarrod), [CrowdIn](https://crowdin.com/profile/breaking_pitt) +- Swedish + - Proofreaders needed. - Turkish - Ali Demirtaş - [GitLab](https://gitlab.com/alidemirtas), [CrowdIn](https://crowdin.com/profile/alidemirtas) - Ukrainian - Volodymyr Sobotovych - [GitLab](https://gitlab.com/wheleph), [CrowdIn](https://crowdin.com/profile/wheleph) - Andrew Vityuk - [GitLab](https://gitlab.com/3_1_3_u), [CrowdIn](https://crowdin.com/profile/andruwa13) - Welsh - - Proofreaders needed. + - Delyth Prys - [GitLab](https://gitlab.com/Delyth), [CrowdIn](https://crowdin.com/profile/DelythPrys) +<!-- vale gitlab.Spelling = YES --> ## Become a proofreader diff --git a/doc/development/i18n/translation.md b/doc/development/i18n/translation.md index e1c02c2c9c2..ed205c20c0d 100644 --- a/doc/development/i18n/translation.md +++ b/doc/development/i18n/translation.md @@ -61,8 +61,12 @@ This helps maintain a logical connection and consistency between tools (e.g. ### Formality -The level of formality used in software varies by language. -For example, in French we translate `you` as the formal `vous`. +The level of formality used in software varies by language: + +| Language | Formality | Example | +| -------- | --------- | ------- | +| French | formal | `vous` for `you` | +| German | informal | `du` for `you` | You can refer to other translated strings and notes in the glossary to assist determining a suitable level of formality. @@ -75,18 +79,22 @@ ethnicity. In languages which distinguish between a male and female form, use both or choose a neutral formulation. +<!-- vale gitlab.Spelling = NO --> For example in German, the word "user" can be translated into "Benutzer" (male) or "Benutzerin" (female). Therefore "create a new user" would translate into "Benutzer(in) anlegen". +<!-- vale gitlab.Spelling = YES --> ### Updating the glossary To propose additions to the glossary please -[open an issue](https://gitlab.com/gitlab-org/gitlab/issues?scope=all&utf8=✓&state=all&label_name[]=Category%3AInternationalization). +[open an issue](https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&utf8=✓&state=all&label_name[]=Category%3AInternationalization). ## French Translation Guidelines ### Inclusive language in French +<!-- vale gitlab.Spelling = NO --> In French, the "écriture inclusive" is now over (see on [Legifrance](https://www.legifrance.gouv.fr/affichTexte.do?cidTexte=JORFTEXT000036068906&categorieLien=id)). So, to include both genders, write “Utilisateurs et utilisatrices” instead of “Utilisateur·rice·s”. When space is missing, the male gender should be used alone. +<!-- vale gitlab.Spelling = YES --> diff --git a/doc/development/img/bullet_v13_0.png b/doc/development/img/bullet_v13_0.png Binary files differnew file mode 100644 index 00000000000..04b476db581 --- /dev/null +++ b/doc/development/img/bullet_v13_0.png diff --git a/doc/development/import_export.md b/doc/development/import_export.md index 6d1b6929667..ecdfd8a853e 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -44,19 +44,34 @@ WARN: Work still in progress <struct with JID> ### Timeouts -Timeout errors occur due to the `StuckImportJobsWorker` marking the process as failed: +Timeout errors occur due to the `Gitlab::Import::StuckProjectImportJobsWorker` marking the process as failed: ```ruby -class StuckImportJobsWorker - include ApplicationWorker - include CronjobQueue - - IMPORT_JOBS_EXPIRATION = 15.hours.to_i +module Gitlab + module Import + class StuckProjectImportJobsWorker + include Gitlab::Import::StuckImportJob + # ... + end + end +end - def perform - import_state_without_jid_count = mark_import_states_without_jid_as_failed! - import_state_with_jid_count = mark_import_states_with_jid_as_failed! - ... +module Gitlab + module Import + module StuckImportJob + # ... + IMPORT_JOBS_EXPIRATION = 15.hours.to_i + # ... + def perform + stuck_imports_without_jid_count = mark_imports_without_jid_as_failed! + stuck_imports_with_jid_count = mark_imports_with_jid_as_failed! + + track_metrics(stuck_imports_with_jid_count, stuck_imports_without_jid_count) + end + # ... + end + end +end ``` ```shell @@ -92,14 +107,14 @@ Marked stuck import jobs as failed. JIDs: xyz While the performance problems are not tackled, there is a process to workaround importing big projects, using a foreground import: -[Foreground import](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/5384) of big projects for customers. +[Foreground import](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/5384) of big projects for customers. (Using the import template in the [infrastructure tracker](https://gitlab.com/gitlab-com/gl-infra/infrastructure/)) ## Security The Import/Export feature is constantly updated (adding new things to export), however the code hasn't been refactored in a long time. We should perform a code audit (see -[confidential issue](../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab/issues/20720`). +[confidential issue](../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab/-/issues/20720`). to make sure its dynamic nature does not increase the number of security concerns. ### Security in the code @@ -192,20 +207,8 @@ module Gitlab ## Version history -The [current version history](../user/project/settings/import_export.md) also displays the equivalent GitLab version -and it is useful for knowing which versions won't be compatible between them. - -| Exporting GitLab version | Importing GitLab version | -| -------------------------- | -------------------------- | -| 11.7 to current | 11.7 to current | -| 11.1 to 11.6 | 11.1 to 11.6 | -| 10.8 to 11.0 | 10.8 to 11.0 | -| 10.4 to 10.7 | 10.4 to 10.7 | -| ... | ... | -| 8.10.3 to 8.11 | 8.10.3 to 8.11 | -| 8.10.0 to 8.10.2 | 8.10.0 to 8.10.2 | -| 8.9.5 to 8.9.11 | 8.9.5 to 8.9.11 | -| 8.9.0 to 8.9.4 | 8.9.0 to 8.9.4 | +Check the [version history](../user/project/settings/import_export.md#version-history) +for compatibility when importing and exporting projects. ### When to bump the version up diff --git a/doc/development/insert_into_tables_in_batches.md b/doc/development/insert_into_tables_in_batches.md index e5c4dc6ee56..d8919789808 100644 --- a/doc/development/insert_into_tables_in_batches.md +++ b/doc/development/insert_into_tables_in_batches.md @@ -187,8 +187,6 @@ There are a few restrictions to how these APIs can be used: - `BulkInsertableAssociations`: - It is currently only compatible with `has_many` relations. - It does not yet support `has_many through: ...` relations. -- Writing [`jsonb`](https://www.postgresql.org/docs/current/datatype-json.html) content is -[not currently supported](https://gitlab.com/gitlab-org/gitlab/-/issues/210560). Moreover, input data should either be limited to around 1000 records at most, or already batched prior to calling bulk insert. The `INSERT` statement will run in a single diff --git a/doc/development/instrumentation.md b/doc/development/instrumentation.md index d72e1c6635e..ee1aab1456e 100644 --- a/doc/development/instrumentation.md +++ b/doc/development/instrumentation.md @@ -119,9 +119,9 @@ without measuring anything. Three values are measured for a block: -- The real time elapsed, stored in NAME_real_time. -- The CPU time elapsed, stored in NAME_cpu_time. -- The call count, stored in NAME_call_count. +- The real time elapsed, stored in `NAME_real_time`. +- The CPU time elapsed, stored in `NAME_cpu_time`. +- The call count, stored in `NAME_call_count`. Both the real and CPU timings are measured in milliseconds. diff --git a/doc/development/integrations/jenkins.md b/doc/development/integrations/jenkins.md index 001d1c21fd3..f2bc6532dde 100644 --- a/doc/development/integrations/jenkins.md +++ b/doc/development/integrations/jenkins.md @@ -1,6 +1,6 @@ # How to run Jenkins in development environment (on macOS) **(STARTER)** -This is a step by step guide on how to set up [Jenkins](https://jenkins.io/) on your local machine and connect to it from your GitLab instance. GitLab triggers webhooks on Jenkins, and Jenkins connects to GitLab using the API. By running both applications on the same machine, we can make sure they are able to access each other. +This is a step by step guide on how to set up [Jenkins](https://www.jenkins.io/) on your local machine and connect to it from your GitLab instance. GitLab triggers webhooks on Jenkins, and Jenkins connects to GitLab using the API. By running both applications on the same machine, we can make sure they are able to access each other. ## Install Jenkins diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index b0e1e28ba8b..1737daae0e0 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -15,7 +15,7 @@ scanner, as well as requirements and guidelines for the Docker image. ## Job definition -This section desribes several important fields to add to the security scanner's job +This section describes several important fields to add to the security scanner's job definition file. Full documentation on these and other available fields can be viewed in the [CI documentation](../../ci/yaml/README.md#image). @@ -69,7 +69,7 @@ For example, here is the definition of a SAST job that generates a file named `g and uploads it as a SAST report: ```yaml -mysec_sast_scanning: +mysec_sast: image: registry.gitlab.com/secure/mysec artifacts: reports: @@ -89,9 +89,9 @@ for variables such as `DEPENDENCY_SCANNING_DISABLED`, `CONTAINER_SCANNING_DISABL disable running the custom scanner. GitLab also defines a `CI_PROJECT_REPOSITORY_LANGUAGES` variable, which provides the list of -languages in the repo. Depending on this value, your scanner may or may not do something different. +languages in the repository. Depending on this value, your scanner may or may not do something different. Language detection currently relies on the [`linguist`](https://github.com/github/linguist) Ruby gem. -See [GitLab CI/CD prefined variables](../../ci/variables/predefined_variables.md#variables-reference). +See [GitLab CI/CD predefined variables](../../ci/variables/predefined_variables.md). #### Policy checking example @@ -124,9 +124,9 @@ regardless of the individual machine the scanner runs on. Depending on the CI infrastructure, the CI may have to fetch the Docker image every time the job runs. -To make the scanning job run fast, and to avoid wasting bandwidth, -it is important to make Docker images as small as possible, -ideally smaller than 50 MB. +For the scanning job to run fast and avoid wasting bandwidth, Docker images should be as small as +possible. You should aim for 50MB or smaller. If that isn't possible, try to keep it below 1.46 GB, +which is the size of a CD-ROM. If the scanner requires a fully functional Linux environment, it is recommended to use a [Debian](https://www.debian.org/intro/about) "slim" distribution or [Alpine Linux](https://www.alpinelinux.org/). @@ -135,11 +135,27 @@ and to compile the scanner with all the libraries it needs. [Multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) might also help with keeping the image small. +To keep an image size small, consider using [dive](https://github.com/wagoodman/dive#dive) to analyze layers in a Docker image to +identify where additional bloat might be originating from. + +In some cases, it might be difficult to remove files from an image. When this occurs, consider using +[Zstandard](https://github.com/facebook/zstd) +to compress files or large directories. Zstandard offers many different compression levels that can +decrease the size of your image with very little impact to decompression speed. It may be helpful to +automatically decompress any compressed directories as soon as an image launches. You can accomplish +this by adding a step to the Docker image's `/etc/bashrc` or to a specific user's `$HOME/.bashrc`. +Remember to change the entry point to launch a bash login shell if you chose the latter option. + +Here are some examples to get you started: + +- <https://gitlab.com/gitlab-org/security-products/license-management/-/blob/0b976fcffe0a9b8e80587adb076bcdf279c9331c/config/install.sh#L168-170> +- <https://gitlab.com/gitlab-org/security-products/license-management/-/blob/0b976fcffe0a9b8e80587adb076bcdf279c9331c/config/.bashrc#L49> + ### Image tag As documented in the [Docker Official Images](https://github.com/docker-library/official-images#tags-and-aliases) project, it is strongly encouraged that version number tags be given aliases which allows the user to easily refer to the "most recent" release of a particular series. -See also [Docker Tagging: Best practices for tagging and versioning docker images](https://docs.microsoft.com/en-us/archive/blogs/stevelasker/docker-tagging-best-practices-for-tagging-and-versioning-docker-images). +See also [Docker Tagging: Best practices for tagging and versioning Docker images](https://docs.microsoft.com/en-us/archive/blogs/stevelasker/docker-tagging-best-practices-for-tagging-and-versioning-docker-images). ## Command line @@ -448,7 +464,7 @@ Right now, GitLab cannot track a vulnerability if its location changes as new Git commits are pushed, and this results in user feedback being lost. For instance, user feedback on a SAST vulnerability is lost if the affected file is renamed or the affected line moves down. -This is addressed in [issue #7586](https://gitlab.com/gitlab-org/gitlab/issues/7586). +This is addressed in [issue #7586](https://gitlab.com/gitlab-org/gitlab/-/issues/7586). In some cases, the multiple scans executed in the same CI pipeline result in duplicates that are automatically merged using the vulnerability location and identifiers. @@ -470,6 +486,10 @@ The confidence ranges from `Low` to `Confirmed`, but it can also be `Unknown`, `Experimental` or even `Ignore` if the vulnerability is to be ignored. Valid values are: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, or `Confirmed` +`Unknown` values means that data is unavailable to determine it's actual value. Therefore, it may be `high`, `medium`, or `low`, +and needs to be investigated. We have [provided a chart](../../user/application_security/sast/analyzers.md#analyzers-data) +of the available SAST Analyzers and what data is currently available. + ### Remediations The `remediations` field of the report is an array of remediation objects. diff --git a/doc/development/integrations/secure_partner_integration.md b/doc/development/integrations/secure_partner_integration.md index 59336b0e6a1..22e1f8bf769 100644 --- a/doc/development/integrations/secure_partner_integration.md +++ b/doc/development/integrations/secure_partner_integration.md @@ -28,7 +28,7 @@ best place to integrate your own product and its results into GitLab. implications for app security, corporate policy, or compliance. When complete, the job reports back on its status and creates a [job artifact](../../user/project/pipelines/job_artifacts.md) as a result. -- The [Merge Request Security Widget](../../user/project/merge_requests/index.md#security-reports-ultimate) +- The [Merge Request Security Widget](../../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports-ultimate) displays the results of the pipeline's security checks and the developer can review them. The developer can review both a summary and a detailed version of the results. @@ -54,10 +54,10 @@ best place to integrate your own product and its results into GitLab. ## How to onboard This section describes the steps you need to complete to onboard as a partner -and complete an intgration with the Secure stage. +and complete an integration with the Secure stage. 1. Read about our [partnerships](https://about.gitlab.com/partners/integrate/). -1. [Create an issue](https://gitlab.com/gitlab-com/alliances/alliances/issues/new?issuable_template=new_partner) +1. [Create an issue](https://gitlab.com/gitlab-com/alliances/alliances/-/issues/new?issuable_template=new_partner) using our new partner issue template to begin the discussion. 1. Get a test account to begin developing your integration. You can request a [GitLab.com Gold Subscription Sandbox](https://about.gitlab.com/partners/integrate/#gitlabcom-gold-subscription-sandbox-request) @@ -76,10 +76,10 @@ and complete an intgration with the Secure stage. - Documentation for [Dependency Scanning reports](../../user/application_security/dependency_scanning/index.md#reports-json-format). - Documentation for [Container Scanning reports](../../user/application_security/container_scanning/index.md#reports-json-format). - See this [example secure job definition that also defines the artifact created](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml). - - If you need a new kind of scan or report, [create an issue](https://gitlab.com/gitlab-org/gitlab/issues/new#) + - If you need a new kind of scan or report, [create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new#) and add the label `devops::secure`. - Once the job is completed, the data can be seen: - - In the [Merge Request Security Report](../../user/project/merge_requests/index.md#security-reports-ultimate) ([MR Security Report data flow](https://gitlab.com/snippets/1910005#merge-request-view)). + - In the [Merge Request Security Report](../../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports-ultimate) ([MR Security Report data flow](https://gitlab.com/snippets/1910005#merge-request-view)). - While [browsing a Job Artifact](../../user/project/pipelines/job_artifacts.md). - In the [Security Dashboard](../../user/application_security/security_dashboard/index.md) ([Dashboard data flow](https://gitlab.com/snippets/1910005#project-and-group-dashboards)). 1. Optional: Provide a way to interact with results as Vulnerabilities: @@ -99,5 +99,9 @@ and complete an intgration with the Secure stage. doing an [Unfiltered blog post](https://about.gitlab.com/handbook/marketing/blog/unfiltered/), doing a co-branded webinar, or producing a co-branded whitepaper. +We have a [video playlist](https://www.youtube.com/playlist?list=PL05JrBw4t0KpMqYxJiOLz-uBIr5w-yP4A) +that may be helpful as part of this process. This covers various topics related to integrating your +tool. + If you have any issues while working through your integration or the steps above, please create an issue to discuss with us further. diff --git a/doc/development/internal_api.md b/doc/development/internal_api.md index 5b53f223eb0..d220a2d46fb 100644 --- a/doc/development/internal_api.md +++ b/doc/development/internal_api.md @@ -43,11 +43,11 @@ POST /internal/allowed | `key_id` | string | no | ID of the SSH-key used to connect to GitLab-shell | | `username` | string | no | Username from the certificate used to connect to GitLab-Shell | | `project` | string | no (if `gl_repository` is passed) | Path to the project | -| `gl_repository` | string | no (if `project` is passed) | Path to the project | +| `gl_repository` | string | no (if `project` is passed) | Repository identifier (e.g. `project-7`) | | `protocol` | string | yes | SSH when called from GitLab-shell, HTTP or SSH when called from Gitaly | | `action` | string | yes | Git command being run (`git-upload-pack`, `git-receive-pack`, `git-upload-archive`) | | `changes` | string | yes | `<oldrev> <newrev> <refname>` when called from Gitaly, The magic string `_any` when called from GitLab Shell | -| `check_ip` | string | no | Ip address from which call to GitLab Shell was made | +| `check_ip` | string | no | IP address from which call to GitLab Shell was made | Example request: diff --git a/doc/development/issue_types.md b/doc/development/issue_types.md index bcd3980c298..028d42b27fc 100644 --- a/doc/development/issue_types.md +++ b/doc/development/issue_types.md @@ -5,9 +5,9 @@ Sometimes when a new resource type is added it's not clear if it should be only (similar to Issue, Epic, Merge Request, Snippet). The idea of Issue Types was first proposed in [this -issue](https://gitlab.com/gitlab-org/gitlab/issues/8767) and its usage was +issue](https://gitlab.com/gitlab-org/gitlab/-/issues/8767) and its usage was discussed few times since then, for example in [incident -management](https://gitlab.com/gitlab-org/gitlab-foss/issues/55532). +management](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55532). ## What is an Issue Type diff --git a/doc/development/licensing.md b/doc/development/licensing.md index e1be1faa61b..f92151e7a37 100644 --- a/doc/development/licensing.md +++ b/doc/development/licensing.md @@ -12,6 +12,8 @@ Some gems may not include their license information in their `gemspec` file, and ### License Finder commands +> Note: License Finder currently uses GitLab misused terms of whitelist and blacklist. As a result, the commands below references those terms. We've created an [issue on their project](https://github.com/pivotal/LicenseFinder/issues/745) to propose that they rename their commands. + There are a few basic commands License Finder provides that you'll need in order to manage license detection. To verify that the checks are passing, and/or to see what dependencies are causing the checks to fail: @@ -20,16 +22,16 @@ To verify that the checks are passing, and/or to see what dependencies are causi bundle exec license_finder ``` -To whitelist a new license: +To allowlist a new license: ```shell license_finder whitelist add MIT ``` -To blacklist a new license: +To denylist a new license: ```shell -license_finder blacklist add GPLv2 +license_finder blacklist add Unlicense ``` To tell License Finder about a dependency's license if it isn't auto-detected: diff --git a/doc/development/logging.md b/doc/development/logging.md index e7d48e4d278..ccd4adf7cef 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -284,7 +284,7 @@ creating visualizations in Kibana. **Note:** The fields of the context are currently only logged for Sidekiq jobs triggered through web requests. See the -[follow-up work](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/68) +[follow-up work](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/68) for more information. ## Exception Handling @@ -358,8 +358,8 @@ end for most users, but you may need to tweak them in [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab). 1. If you add a new file, submit an issue to the [production - tracker](https://gitlab.com/gitlab-com/gl-infra/production/issues) or - a merge request to the [gitlab_fluentd](https://gitlab.com/gitlab-cookbooks/gitlab_fluentd) + tracker](https://gitlab.com/gitlab-com/gl-infra/production/-/issues) or + a merge request to the [`gitlab_fluentd`](https://gitlab.com/gitlab-cookbooks/gitlab_fluentd) project. See [this example](https://gitlab.com/gitlab-cookbooks/gitlab_fluentd/-/merge_requests/51/diffs). 1. Be sure to update the [GitLab CE/EE documentation](../administration/logs.md) and the [GitLab.com diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index b51fc681e27..ed6d08a9894 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -193,7 +193,7 @@ in Puma/Unicorn threads. Often, GitLab needs to communicate with an external service such as Kubernetes clusters. In this case, it's hard to estimate when the external service finishes the requested process, for example, if it's a user-owned cluster that's inactive for some reason, -GitLab might wait for the response forever ([Example](https://gitlab.com/gitlab-org/gitlab/issues/31475)). +GitLab might wait for the response forever ([Example](https://gitlab.com/gitlab-org/gitlab/-/issues/31475)). This could result in Puma/Unicorn timeout and should be avoided at all cost. You should set a reasonable timeout, gracefully handle exceptions and surface the @@ -210,7 +210,7 @@ as an open transaction basically blocks the release of a PostgreSQL backend conn For keeping transaction as minimal as possible, please consider using `AfterCommitQueue` module or `after_commit` AR hook. -Here is [an example](https://gitlab.com/gitlab-org/gitlab/issues/36154#note_247228859) +Here is [an example](https://gitlab.com/gitlab-org/gitlab/-/issues/36154#note_247228859) that one request to Gitaly instance during transaction triggered a P1 issue. ## Eager Loading diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 4cf546173de..7d3d9dac174 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -35,7 +35,7 @@ and post-deployment migrations (`db/post_migrate`) are run after the deployment ## Schema Changes -Changes to the schema should be commited to `db/structure.sql`. This +Changes to the schema should be committed to `db/structure.sql`. This file is automatically generated by Rails, so you normally should not edit this file by hand. If your migration is adding a column to a table, that column will be added at the bottom. Please do not reorder @@ -49,7 +49,7 @@ regenerate a clean `db/structure.sql` for the migrations you're adding. This script will apply all migrations found in `db/migrate` or `db/post_migrate`, so if there are any migrations you don't want to commit to the schema, rename or remove them. If your branch is not -targetting `master` you can set the `TARGET` environment variable. +targeting `master` you can set the `TARGET` environment variable. ```shell # Regenerate schema against `master` @@ -343,7 +343,7 @@ def up end ``` -The RuboCop rule generally allows standard Rails migration methods, listed below. This example will cause a rubocop offense: +The RuboCop rule generally allows standard Rails migration methods, listed below. This example will cause a Rubocop offense: ```ruby disabled_ddl_transaction! @@ -367,6 +367,8 @@ migration involves one of the high-traffic tables: - `users` - `projects` - `namespaces` +- `issues` +- `merge_requests` - `ci_pipelines` - `ci_builds` - `notes` @@ -550,6 +552,12 @@ operations that don't require `disable_ddl_transaction!`. You can read more about adding [foreign key constraints to an existing column](database/add_foreign_key_to_existing_column.md). +## `NOT NULL` constraints + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38358) in GitLab 13.0. + +See the style guide on [`NOT NULL` constraints](database/not_null_constraints.md) for more information. + ## Adding Columns With Default Values With PostgreSQL 11 being the minimum version since GitLab 13.0, adding columns with default values has become much easier and @@ -559,6 +567,11 @@ Before PostgreSQL 11, adding a column with a default was problematic as it would have caused a full table rewrite. The corresponding helper `add_column_with_default` has been deprecated and will be removed in a later release. +NOTE: **Note:** +If a backport adding a column with a default value is needed for %12.9 or earlier versions, +it should use `add_column_with_default` helper. If a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3) +is involved, backporting to %12.9 is contraindicated. + ## Changing the column default One might think that changing a default column with `change_column_default` is an @@ -611,7 +624,7 @@ end ``` If a computed update is needed, the value can be wrapped in `Arel.sql`, so Arel -treats it as an SQL literal. It's also a required deprecation for [Rails 6](https://gitlab.com/gitlab-org/gitlab/issues/28497). +treats it as an SQL literal. It's also a required deprecation for [Rails 6](https://gitlab.com/gitlab-org/gitlab/-/issues/28497). The below example is the same as the one above, but the value is set to the product of the `bar` and `baz` columns: @@ -727,6 +740,12 @@ Rails migration example: add_column(:projects, :foo, :integer, default: 10, limit: 8) ``` +## Strings and the Text data type + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30453) in GitLab 13.0. + +See the [text data type](database/strings_and_the_text_data_type.md) style guide for more information. + ## Timestamp column type By default, Rails uses the `timestamp` data type that stores timestamp data diff --git a/doc/development/module_with_instance_variables.md b/doc/development/module_with_instance_variables.md index b0eab95190b..2229c99c99b 100644 --- a/doc/development/module_with_instance_variables.md +++ b/doc/development/module_with_instance_variables.md @@ -30,11 +30,11 @@ People are saying multiple inheritance is bad. Mixing multiple modules with multiple instance variables scattering everywhere suffer from the same issue. The same applies to `ActiveSupport::Concern`. See: [Consider replacing concerns with dedicated classes & composition]( -https://gitlab.com/gitlab-org/gitlab/issues/16270) +https://gitlab.com/gitlab-org/gitlab/-/issues/16270) There's also a similar idea: [Use decorators and interface segregation to solve overgrowing models problem]( -https://gitlab.com/gitlab-org/gitlab/issues/14235) +https://gitlab.com/gitlab-org/gitlab/-/issues/14235) Note that `included` doesn't solve the whole issue. They define the dependencies, but they still allow each modules to talk implicitly via the diff --git a/doc/development/multi_version_compatibility.md b/doc/development/multi_version_compatibility.md index aedd5c1ffb7..001517d44ea 100644 --- a/doc/development/multi_version_compatibility.md +++ b/doc/development/multi_version_compatibility.md @@ -45,7 +45,7 @@ and set this column to `false`. The old servers were still updating the old colu that updated the new column from the old one. For the new servers though, they were only updating the new column and that same trigger was now working against us and setting it back to the wrong value. -For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9176). +For more information, see this [confidential issue](../user/project/issues/confidential_issues.md) `https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9176`. ### Sidebar wasn't loading for some users diff --git a/doc/development/namespaces_storage_statistics.md b/doc/development/namespaces_storage_statistics.md index 3065d4f84a2..5207276ba73 100644 --- a/doc/development/namespaces_storage_statistics.md +++ b/doc/development/namespaces_storage_statistics.md @@ -25,7 +25,7 @@ by [`Namespaces#with_statistics`](https://gitlab.com/gitlab-org/gitlab/blob/4ab5 Additionally, the pattern that is currently used to update the project statistics (the callback) doesn't scale adequately. It is currently one of the largest -[database queries transactions on production](https://gitlab.com/gitlab-org/gitlab/issues/29070) +[database queries transactions on production](https://gitlab.com/gitlab-org/gitlab/-/issues/29070) that takes the most time overall. We can't add one more query to it as it will increase the transaction's length. @@ -142,7 +142,7 @@ but we refresh them through Sidekiq jobs and in different transactions: 1. Create a second table (`namespace_aggregation_schedules`) with two columns `id` and `namespace_id`. 1. Whenever the statistics of a project changes, insert a row into `namespace_aggregation_schedules` - We don't insert a new row if there's already one related to the root namespace. - - Keeping in mind the length of the transaction that involves updating `project_statistics`(<https://gitlab.com/gitlab-org/gitlab/issues/29070>), the insertion should be done in a different transaction and through a Sidekiq Job. + - Keeping in mind the length of the transaction that involves updating `project_statistics`(<https://gitlab.com/gitlab-org/gitlab/-/issues/29070>), the insertion should be done in a different transaction and through a Sidekiq Job. 1. After inserting the row, we schedule another worker to be executed async at two different moments: - One enqueued for immediate execution and another one scheduled in `1.5h` hours. - We only schedule the jobs, if we can obtain a `1.5h` lease on Redis on a key based on the root namespace ID. @@ -162,7 +162,7 @@ This implementation has the following benefits: The only downside of this approach is that namespaces' statistics are updated up to `1.5` hours after the change is done, which means there's a time window in which the statistics are inaccurate. Because we're still not -[enforcing storage limits](https://gitlab.com/gitlab-org/gitlab/issues/17664), this is not a major problem. +[enforcing storage limits](https://gitlab.com/gitlab-org/gitlab/-/issues/17664), this is not a major problem. ## Conclusion @@ -171,8 +171,8 @@ performant approach of aggregating the root namespaces. All the details regarding this use case can be found on: -- <https://gitlab.com/gitlab-org/gitlab-foss/issues/62214> +- <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62214> - Merge Request with the implementation: <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28996> Performance of the namespace storage statistics were measured in staging and production (GitLab.com). All results were posted -on <https://gitlab.com/gitlab-org/gitlab-foss/issues/64092>: No problem has been reported so far. +on <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64092>: No problem has been reported so far. diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md index aa76a9fec07..f76fd72d4dc 100644 --- a/doc/development/new_fe_guide/development/accessibility.md +++ b/doc/development/new_fe_guide/development/accessibility.md @@ -1,4 +1,4 @@ -# Accessiblity +# Accessibility Using semantic HTML plays a key role when it comes to accessibility. @@ -37,7 +37,7 @@ In forms we should use the `for` attribute in the label statement: ## Testing 1. On MacOS you can use [VoiceOver](https://www.apple.com/accessibility/mac/vision/) by pressing `cmd+F5`. -1. On Windows you can use [Narrator](https://www.microsoft.com/en-us/accessibility/windows) by pressing Windows logo key + Ctrl + Enter. +1. On Windows you can use [Narrator](https://www.microsoft.com/en-us/accessibility/windows) by pressing Windows logo key + Control + Enter. ## Online resources diff --git a/doc/development/new_fe_guide/development/performance.md b/doc/development/new_fe_guide/development/performance.md index 4a85c04d8cf..7b58a80576a 100644 --- a/doc/development/new_fe_guide/development/performance.md +++ b/doc/development/new_fe_guide/development/performance.md @@ -5,11 +5,11 @@ We have a performance dashboard available in one of our [Grafana instances](https://dashboards.gitlab.net/d/1EBTz3Dmz/sitespeed-page-summary?orgId=1). This dashboard automatically aggregates metric data from [sitespeed.io](https://www.sitespeed.io/) every 6 hours. These changes are displayed after a set number of pages are aggregated. These pages can be found inside a text file in the [`gitlab-build-images` repository](https://gitlab.com/gitlab-org/gitlab-build-images) called [`gitlab.txt`](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/scripts/gitlab.txt) -Any frontend engineer can contribute to this dashboard. They can contribute by adding or removing urls of pages from this text file. Please have a [frontend monitoring expert](https://about.gitlab.com/company/team/) review your changes before assigning to a maintainer of the `gitlab-build-images` project. The changes will go live on the next scheduled run after the changes are merged into `master`. +Any frontend engineer can contribute to this dashboard. They can contribute by adding or removing URLs of pages from this text file. Please have a [frontend monitoring expert](https://about.gitlab.com/company/team/) review your changes before assigning to a maintainer of the `gitlab-build-images` project. The changes will go live on the next scheduled run after the changes are merged into `master`. There are 3 recommended high impact metrics to review on each page: -- [First visual change](https://developers.google.com/web/tools/lighthouse/audits/first-meaningful-paint) +- [First visual change](https://web.dev/first-meaningful-paint/) - [Speed Index](https://sites.google.com/a/webpagetest.org/docs/using-webpagetest/metrics/speed-index) - [Visual Complete 95%](https://sites.google.com/a/webpagetest.org/docs/using-webpagetest/metrics/speed-index) diff --git a/doc/development/omnibus.md b/doc/development/omnibus.md index 28ca500f21a..deaf72d2ecf 100644 --- a/doc/development/omnibus.md +++ b/doc/development/omnibus.md @@ -24,7 +24,7 @@ and write it to the Rails root. In the Omnibus packages, reconfigure writes the The Omnibus design separates code (read-only, under `/opt/gitlab`) from data (read/write, under `/var/opt/gitlab`) and logs (read/write, under `/var/log/gitlab`). To make this happen the reconfigure script sets custom -paths where it can in GitLab config files, and where there are no path +paths where it can in GitLab configuration files, and where there are no path settings, it uses symlinks. For example, `config/gitlab.yml` is treated as data so that file is a symlink. diff --git a/doc/development/packages.md b/doc/development/packages.md index 11fc3faf4ab..edf01255e84 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -10,7 +10,7 @@ not cover the way the code should be written. However, you can find a good examp by looking at merge requests with Maven and NPM support: - [NPM registry support](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8673). -- [Conan repository](https://gitlab.com/gitlab-org/gitlab/issues/8248). +- [Conan repository](https://gitlab.com/gitlab-org/gitlab/-/issues/8248). - [Maven repository](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6607). - [Instance level endpoint for Maven repository](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8757) @@ -61,15 +61,19 @@ The current state of existing package registries availability is: | Repository Type | Project Level | Group Level | Instance Level | |-----------------|---------------|-------------|----------------| | Maven | Yes | Yes | Yes | -| Conan | No - [open issue](https://gitlab.com/gitlab-org/gitlab/issues/11679) | No - [open issue](https://gitlab.com/gitlab-org/gitlab/issues/11679) | Yes | -| NPM | No - [open issue](https://gitlab.com/gitlab-org/gitlab/issues/36853) | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/issues/36853) | +| Conan | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) | Yes | +| NPM | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36853) | | NuGet | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/36423) | No | | PyPI | Yes | No | No | +| Go | Yes | No - [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213900) | No - [open-issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213902) | +| Composer | Yes | Yes | No | NOTE: **Note:** NPM is currently a hybrid of the instance level and group level. It is using the top-level group or namespace as the defining portion of the name (for example, `@my-group-name/my-package-name`). +**Note:** Composer package naming scope is Instance Level. + ### Naming conventions To avoid name conflict for instance-level endpoints you will need to define a package naming convention diff --git a/doc/development/performance.md b/doc/development/performance.md index 2e73161a11f..69ad524675d 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -8,7 +8,7 @@ consistent performance of GitLab. The process of solving performance problems is roughly as follows: 1. Make sure there's an issue open somewhere (for example, on the GitLab CE issue - tracker), and create one if there is not. See [#15607](https://gitlab.com/gitlab-org/gitlab-foss/issues/15607) for an example. + tracker), and create one if there is not. See [#15607](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/15607) for an example. 1. Measure the performance of the code in a production environment such as GitLab.com (see the [Tooling](#tooling) section below). Performance should be measured over a period of _at least_ 24 hours. @@ -42,6 +42,7 @@ GitLab provides built-in tools to help improve performance and availability: - [Request Profiling](../administration/monitoring/performance/request_profiling.md). - [QueryRecoder](query_recorder.md) for preventing `N+1` regressions. - [Chaos endpoints](chaos_endpoints.md) for testing failure scenarios. Intended mainly for testing availability. +- [Service measurement](service_measurement.md) for measuring and logging service execution. GitLab team members can use [GitLab.com's performance monitoring systems](https://about.gitlab.com/handbook/engineering/monitoring/) located at <https://dashboards.gitlab.net>, this requires you to log in using your diff --git a/doc/development/permissions.md b/doc/development/permissions.md index 0772389bf9e..06a4a03de38 100644 --- a/doc/development/permissions.md +++ b/doc/development/permissions.md @@ -14,7 +14,7 @@ Groups and projects can have the following visibility levels: - private (`0`) - an entity is visible only to the approved members of the entity The visibility level of a group can be changed only if all subgroups and -subprojects have the same or lower visibility level. (e.g., a group can be set +sub-projects have the same or lower visibility level. (e.g., a group can be set to internal only if all subgroups and projects are internal or private). Visibility levels can be found in the `Gitlab::VisibilityLevel` module. @@ -41,11 +41,12 @@ can be accessed only by project members by default. Users can be members of multiple groups and projects. The following access levels are available (defined in the `Gitlab::Access` module): -- Guest -- Reporter -- Developer -- Maintainer -- Owner +- No access (`0`) +- Guest (`10`) +- Reporter (`20`) +- Developer (`30`) +- Maintainer (`40`) +- Owner (`50`) If a user is the member of both a project and the project parent group, the higher permission is taken into account for the project. @@ -56,6 +57,12 @@ can still view the groups and their entities (like epics). Project membership (where the group membership is already taken into account) is stored in the `project_authorizations` table. +CAUTION: **Caution:** +Due to [an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219299), +projects in personal namespace will not show owner (`50`) permission in +`project_authorizations` table. Note however that [`user.owned_projects`](https://gitlab.com/gitlab-org/gitlab/blob/0d63823b122b11abd2492bca47cc26858eee713d/app/models/user.rb#L906-916) +is calculated properly. + ### Confidential issues Confidential issues can be accessed only by project members who are at least @@ -92,10 +99,10 @@ into different features like Merge Requests and CI flow. | Activity level | Resource | Locations |Permission dependency| |----------------|----------|-----------|-----| -| View | License information | Dependency list, License Compliance | Can view repo | -| View | Dependency information | Dependency list, License Compliance | Can view repo | +| View | License information | Dependency list, License Compliance | Can view repository | +| View | Dependency information | Dependency list, License Compliance | Can view repository | | View | Vulnerabilities information | Dependency list | Can view security findings | -| View | Black/Whitelisted licenses for the project | License Compliance, Merge request | Can view repo | +| View | Black/Whitelisted licenses for the project | License Compliance, Merge request | Can view repository | | View | Security findings | Merge Request, CI job page, Pipeline security tab | Can read the project and CI jobs | | View | Vulnerability feedback | Merge Request | Can read security findings | | View | Dependency List page | Project | Can access Dependency information | diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 39ca846c1cc..05b80cdb4a6 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -11,173 +11,15 @@ We're striving to [dogfood](https://about.gitlab.com/handbook/engineering/#dogfo GitLab [CI/CD features and best-practices](../ci/yaml/README.md) as much as possible. -## Stages +## Overview -The current stages are: - -- `sync`: This stage is used to synchronize changes from <https://gitlab.com/gitlab-org/gitlab> to - <https://gitlab.com/gitlab-org/gitlab-foss>. -- `prepare`: This stage includes jobs that prepare artifacts that are needed by - jobs in subsequent stages. -- `build-images`: This stage includes jobs that prepare docker images - that are needed by jobs in subsequent stages or downstream pipelines. -- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests. -- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs. -- `post-test`: This stage includes jobs that build reports or gather data from - the `test` stage's jobs (e.g. coverage, Knapsack metadata etc.). -- `review-prepare`: This stage includes a job that build the CNG images that are - later used by the (Helm) Review App deployment (see - [Review Apps](testing_guide/review_apps.md) for details). -- `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps. -- `qa`: This stage includes jobs that perform QA tasks against the Review App - that is deployed in the previous stage. -- `post-qa`: This stage includes jobs that build reports or gather data from - the `qa` stage's jobs (e.g. Review App performance report). -- `pages`: This stage includes a job that deploys the various reports as - GitLab Pages (e.g. <https://gitlab-org.gitlab.io/gitlab/coverage-ruby/>, - <https://gitlab-org.gitlab.io/gitlab/coverage-javascript/>, - <https://gitlab-org.gitlab.io/gitlab/webpack-report/>). - -## Default image - -The default image is defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>. - -It includes Ruby, Go, Git, Git LFS, Chrome, Node, Yarn, PostgreSQL, and Graphics Magick. - -The images used in our pipelines are configured in the -[`gitlab-org/gitlab-build-images`](https://gitlab.com/gitlab-org/gitlab-build-images) -project, which is push-mirrored to <https://dev.gitlab.org/gitlab/gitlab-build-images> -for redundancy. - -The current version of the build images can be found in the -["Used by GitLab section"](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml). - -## Default variables - -In addition to the [predefined variables](../ci/variables/predefined_variables.md), -each pipeline includes default variables defined in -<https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>. - -## Common job definitions - -Most of the jobs [extend from a few CI definitions](../ci/yaml/README.md#extends) -defined in [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml) -that are scoped to a single [configuration parameter](../ci/yaml/README.md#configuration-parameters). - -| Job definitions | Description | -|------------------|-------------| -| `.default-tags` | Ensures a job has the `gitlab-org` tag to ensure it's using our dedicated runners. | -| `.default-retry` | Allows a job to [retry](../ci/yaml/README.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. | -| `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (e.g. tests). | -| `.default-cache` | Allows a job to use a default `cache` definition suitable for Ruby/Rails and frontend tasks. | -| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. | -| `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. | -| `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. | -| `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. | - -## `workflow:rules` - -We're using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) to -define default rules to determine whether or not a pipeline is created. - -These rules are defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml> -and are as follows: - -1. If `$FORCE_GITLAB_CI` is set, create a pipeline. -1. For merge requests, create a pipeline. -1. For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.). -1. For tags, create a pipeline. -1. If `$GITLAB_INTERNAL` isn't set, don't create a pipeline. -1. For stable, auto-deploy, and security branches, create a pipeline. -1. For any other cases (e.g. when pushing a branch with no MR for it), no pipeline is created. - -## `rules`, `if:` conditions and `changes:` patterns - -We're using the [`rules` keyword](../ci/yaml/README.md#rules) extensively. - -All `rules` definitions are defined in -<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>, -then included in individual jobs via [`extends`](../ci/yaml/README.md#extends). - -The `rules` definitions are composed of `if:` conditions and `changes:` patterns, -which are also defined in -<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml> -and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anchors) - -### `if:` conditions - -| `if:` conditions | Description | Notes | -|------------------|-------------|-------| -| `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). | -| `if-not-ee` | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). | -| `if-not-foss` | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). | -| `if-default-refs` | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs won't be created for branches with this default configuration. | -| `if-master-refs` | Matches if the current branch is `master`. | | -| `if-master-or-tag` | Matches if the pipeline is for the `master` branch or for a tag. | | -| `if-merge-request` | Matches if the pipeline is for a merge request. | | -| `if-nightly-master-schedule` | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | | -| `if-dot-com-gitlab-org-schedule` | Limits jobs creation to scheduled pipelines for the `gitlab-org` group on GitLab.com. | | -| `if-dot-com-gitlab-org-master` | Limits jobs creation to the `master` branch for the `gitlab-org` group on GitLab.com. | | -| `if-dot-com-gitlab-org-merge-request` | Limits jobs creation to merge requests for the `gitlab-org` group on GitLab.com. | | -| `if-dot-com-gitlab-org-and-security-tag` | Limits job creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | -| `if-dot-com-gitlab-org-and-security-merge-request` | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | -| `if-dot-com-ee-schedule` | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | | -| `if-cache-credentials-schedule` | Limits jobs to scheduled pipelines with the `$CI_REPO_CACHE_CREDENTIALS` variable set. | | - -### `changes:` patterns - -| `changes:` patterns | Description | -|------------------------------|--------------------------------------------------------------------------| -| `ci-patterns` | Only create job for CI config-related changes. | -| `yaml-patterns` | Only create job for YAML-related changes. | -| `docs-patterns` | Only create job for docs-related changes. | -| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. | -| `frontend-patterns` | Only create job for frontend-related changes. | -| `backstage-patterns` | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). | -| `code-patterns` | Only create job for code-related changes. | -| `qa-patterns` | Only create job for QA-related changes. | -| `code-backstage-patterns` | Combination of `code-patterns` and `backstage-patterns`. | -| `code-qa-patterns` | Combination of `code-patterns` and `qa-patterns`. | -| `code-backstage-qa-patterns` | Combination of `code-patterns`, `backstage-patterns`, and `qa-patterns`. | - -## Interruptible jobs pipelines - -By default, all jobs are [interruptible](../ci/yaml/README.md#interruptible), except the -`dont-interrupt-me` job which runs automatically on `master`, and is `manual` -otherwise. - -If you want a running pipeline to finish even if you push new commits to a merge -request, be sure to start the `dont-interrupt-me` job before pushing. - -## PostgreSQL versions testing - -### Current versions testing - -| Where? | PostgreSQL version | -| ------ | ------ | -| MRs | 11 | -| `master` (non-scheduled pipelines) | 11 | -| 2-hourly scheduled pipelines | 11 | - -### Long-term plan - -We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html): - -| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) | -| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- | -| PG9.6 | MRs/`master`/`2-hour`/`nightly` | - | - | - | - | - | - | - | -| PG10 | `nightly` | - | - | - | - | - | - | - | -| PG11 | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - | -| PG12 | - | - | - | - | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | `master`/`2-hour` | -| PG13 | - | - | - | - | - | - | - | MRs/`master`/`2-hour`/`nightly` | - -## Pipeline types +### Pipeline types Since we use the [`rules:`](../ci/yaml/README.md#rules) and [`needs:`](../ci/yaml/README.md#needs) keywords extensively, we have four main pipeline types which are described below. Note that an MR that includes multiple types of changes would have a pipelines that include jobs from multiple types (e.g. a combination of docs-only and code-only pipelines). -### Docs-only MR pipeline +#### Docs-only MR pipeline Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/135236627> @@ -191,7 +33,7 @@ graph LR end ``` -### Code-only MR pipeline +#### Code-only MR pipeline Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/136295694> @@ -204,11 +46,11 @@ graph RL; click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" 1-2["build-qa-image (3.4 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-3["compile-assets pull-cache (9.06 minutes)"]; + 1-3["compile-test-assets (9.06 minutes)"]; click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" - 1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"]; + 1-4["compile-test-assets as-if-foss (8.35 minutes)"]; click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" - 1-5["gitlab:assets:compile pull-cache (22 minutes)"]; + 1-5["compile-production-assets (22 minutes)"]; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" 1-6["setup-test-env (8.22 minutes)"]; click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" @@ -224,6 +66,8 @@ graph RL; 1-18["kubesec-sast"]; 1-19["nodejs-scan-sast"]; 1-20["secrets-sast"]; + 1-21["static-analysis (17 minutes)"]; + click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" class 1-3 criticalPath; class 1-6 criticalPath; @@ -241,8 +85,6 @@ graph RL; 2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6; end - 2_2-1["static-analysis (17 minutes)"]; - click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" 2_2-2["frontend-fixtures (17.2 minutes)"]; class 2_2-2 criticalPath; click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0" @@ -252,13 +94,13 @@ graph RL; click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0" 2_2-5["webpack-dev-server (6.1 minutes)"]; click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0" - subgraph "Needs `setup-test-env` & `compile-assets`"; - 2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; + subgraph "Needs `setup-test-env` & `compile-test-assets`"; + 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; 2_2-3 --> 1-6 & 1-4; end 2_3-1["build-assets-image (2.5 minutes)"]; - subgraph "Needs `gitlab:assets:compile`"; + subgraph "Needs `compile-production-assets`"; 2_3-1 --> 1-5 end @@ -269,7 +111,7 @@ graph RL; end 2_5-1["rspec & db jobs (12-22 minutes)"]; - subgraph "Needs `compile-assets`, `setup-test-env`, & `retrieve-tests-metadata`"; + subgraph "Needs `compile-test-assets`, `setup-test-env`, & `retrieve-tests-metadata`"; 2_5-1 --> 1-3 & 1-6 & 1-14; class 2_5-1 criticalPath; click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations" @@ -304,7 +146,7 @@ graph RL; end ``` -### Frontend-only MR pipeline +#### Frontend-only MR pipeline Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134661039> @@ -317,11 +159,11 @@ graph RL; click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" 1-2["build-qa-image (3.4 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-3["compile-assets pull-cache (9.06 minutes)"]; + 1-3["compile-test-assets (9.06 minutes)"]; click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" - 1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"]; + 1-4["compile-test-assets as-if-foss (8.35 minutes)"]; click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" - 1-5["gitlab:assets:compile pull-cache (22 minutes)"]; + 1-5["compile-production-assets (22 minutes)"]; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" 1-6["setup-test-env (8.22 minutes)"]; click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" @@ -337,6 +179,8 @@ graph RL; 1-18["kubesec-sast"]; 1-19["nodejs-scan-sast"]; 1-20["secrets-sast"]; + 1-21["static-analysis (17 minutes)"]; + click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" class 1-3 criticalPath; class 1-5 criticalPath; @@ -355,8 +199,6 @@ graph RL; 2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6; end - 2_2-1["static-analysis (17 minutes)"]; - click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" 2_2-2["frontend-fixtures (17.2 minutes)"]; class 2_2-2 criticalPath; click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0" @@ -366,14 +208,14 @@ graph RL; click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0" 2_2-5["webpack-dev-server (6.1 minutes)"]; click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0" - subgraph "Needs `setup-test-env` & `compile-assets`"; - 2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; + subgraph "Needs `setup-test-env` & `compile-test-assets`"; + 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3; 2_2-3 --> 1-6 & 1-4; end 2_3-1["build-assets-image (2.5 minutes)"]; class 2_3-1 criticalPath; - subgraph "Needs `gitlab:assets:compile`"; + subgraph "Needs `compile-production-assets`"; 2_3-1 --> 1-5 end @@ -384,7 +226,7 @@ graph RL; end 2_5-1["rspec & db jobs (12-22 minutes)"]; - subgraph "Needs `compile-assets`, `setup-test-env, & `retrieve-tests-metadata`"; + subgraph "Needs `compile-test-assets`, `setup-test-env, & `retrieve-tests-metadata`"; 2_5-1 --> 1-3 & 1-6 & 1-14; class 2_5-1 criticalPath; click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations" @@ -444,7 +286,7 @@ graph RL; end ``` -### QA-only MR pipeline +#### QA-only MR pipeline Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134645109> @@ -457,11 +299,11 @@ graph RL; click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0" 1-2["build-qa-image (3.4 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-3["compile-assets pull-cache (9.06 minutes)"]; + 1-3["compile-test-assets (9.06 minutes)"]; click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" - 1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"]; + 1-4["compile-test-assets as-if-foss (8.35 minutes)"]; click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0" - 1-5["gitlab:assets:compile pull-cache (22 minutes)"]; + 1-5["compile-production-assets (22 minutes)"]; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" 1-6["setup-test-env (8.22 minutes)"]; click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" @@ -477,6 +319,8 @@ graph RL; 1-18["kubesec-sast"]; 1-19["nodejs-scan-sast"]; 1-20["secrets-sast"]; + 1-21["static-analysis (17 minutes)"]; + click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" class 1-5 criticalPath; end @@ -487,14 +331,8 @@ graph RL; 2_1-1 --> 1-6; end - 2_2-1["static-analysis (17 minutes)"]; - click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0" - subgraph "Needs `setup-test-env` & `compile-assets`"; - 2_2-1 --> 1-6 & 1-3; - end - 2_3-1["build-assets-image (2.5 minutes)"]; - subgraph "Needs `gitlab:assets:compile`"; + subgraph "Needs `compile-production-assets`"; 2_3-1 --> 1-5 class 2_3-1 criticalPath; end @@ -507,24 +345,61 @@ graph RL; end ``` -## Test jobs +### `workflow:rules` + +We're using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) to +define default rules to determine whether or not a pipeline is created. + +These rules are defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml> +and are as follows: + +1. If `$FORCE_GITLAB_CI` is set, create a pipeline. +1. For merge requests, create a pipeline. +1. For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.). +1. For tags, create a pipeline. +1. If `$GITLAB_INTERNAL` isn't set, don't create a pipeline. +1. For stable, auto-deploy, and security branches, create a pipeline. +1. For any other cases (e.g. when pushing a branch with no MR for it), no pipeline is created. + +### PostgreSQL versions testing + +#### Current versions testing + +| Where? | PostgreSQL version | +| ------ | ------ | +| MRs | 11 | +| `master` (non-scheduled pipelines) | 11 | +| 2-hourly scheduled pipelines | 11 | + +#### Long-term plan + +We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html): + +| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) | +| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- | +| PG9.6 | MRs/`master`/`2-hour`/`nightly` | - | - | - | - | - | - | - | +| PG10 | `nightly` | - | - | - | - | - | - | - | +| PG11 | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - | +| PG12 | - | - | - | - | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | `master`/`2-hour` | +| PG13 | - | - | - | - | - | - | - | MRs/`master`/`2-hour`/`nightly` | + +### Test jobs Consult [GitLab tests in the Continuous Integration (CI) context](testing_guide/ci.md) for more information. -## Review app jobs +### Review app jobs Consult the [Review Apps](testing_guide/review_apps.md) dedicated page for more information. -## As-if-FOSS jobs +### As-if-FOSS jobs The `* as-if-foss` jobs allows to run GitLab's test suite "as-if-FOSS", meaning as if the jobs would run in the context of the `gitlab-org/gitlab-foss` project. These jobs are only created in the following cases: -- `master` commits (pushes and scheduled pipelines). - `gitlab-org/security/gitlab` merge requests. - Merge requests which include `RUN AS-IF-FOSS` in their title. -- Merge requests that changes the CI config. +- Merge requests that changes the CI configuration. The `* as-if-foss` jobs have the `FOSS_ONLY='1'` variable set and gets their EE-specific folders removed before the tests start running. @@ -532,9 +407,41 @@ folders removed before the tests start running. The intent is to ensure that a change won't introduce a failure once the `gitlab-org/gitlab` project will be synced to the `gitlab-org/gitlab-foss` project. -## Pre-clone step +## Performance -The `gitlab-org/gitlab` project on GitLab.com uses a [pre-clone step](https://gitlab.com/gitlab-org/gitlab/issues/39134) +### Interruptible pipelines + +By default, all jobs are [interruptible](../ci/yaml/README.md#interruptible), except the +`dont-interrupt-me` job which runs automatically on `master`, and is `manual` +otherwise. + +If you want a running pipeline to finish even if you push new commits to a merge +request, be sure to start the `dont-interrupt-me` job before pushing. + +### Caching strategy + +1. All jobs must only pull caches by default. +1. All jobs must be able to pass with an empty cache. In other words, caches are only there to speed up jobs. +1. We currently have 6 different caches defined in + [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml), + with fixed keys: + - `.rails-cache`. + - `.static-analysis-cache`. + - `.qa-cache` + - `.yarn-cache`. + - `.assets-compile-cache` (the key includes `${NODE_ENV}` so it's actually two different caches). +1. Only 6 specific jobs, running in 2-hourly scheduled pipelines, are pushing (i.e. updating) to the caches: + - `update-rails-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml). + - `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml). + - `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/qa.gitlab-ci.yml). + - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). + - `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). + - `update-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). +1. These jobs will run in merge requests whose title include `UPDATE CACHE`. + +### Pre-clone step + +The `gitlab-org/gitlab` project on GitLab.com uses a [pre-clone step](https://gitlab.com/gitlab-org/gitlab/-/issues/39134) to seed the project with a recent archive of the repository. This is done for several reasons: @@ -597,6 +504,124 @@ credentials are stored in the 1Password GitLab.com Production vault. Note that this bucket should be located in the same continent as the runner, or [network egress charges will apply](https://cloud.google.com/storage/pricing). +## CI configuration internals + +### Stages + +The current stages are: + +- `sync`: This stage is used to synchronize changes from <https://gitlab.com/gitlab-org/gitlab> to + <https://gitlab.com/gitlab-org/gitlab-foss>. +- `prepare`: This stage includes jobs that prepare artifacts that are needed by + jobs in subsequent stages. +- `build-images`: This stage includes jobs that prepare Docker images + that are needed by jobs in subsequent stages or downstream pipelines. +- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests. +- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs. +- `post-test`: This stage includes jobs that build reports or gather data from + the `test` stage's jobs (e.g. coverage, Knapsack metadata etc.). +- `review-prepare`: This stage includes a job that build the CNG images that are + later used by the (Helm) Review App deployment (see + [Review Apps](testing_guide/review_apps.md) for details). +- `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps. +- `qa`: This stage includes jobs that perform QA tasks against the Review App + that is deployed in the previous stage. +- `post-qa`: This stage includes jobs that build reports or gather data from + the `qa` stage's jobs (e.g. Review App performance report). +- `pages`: This stage includes a job that deploys the various reports as + GitLab Pages (e.g. [`coverage-ruby`](https://gitlab-org.gitlab.io/gitlab/coverage-ruby/), + [`coverage-javascript`](https://gitlab-org.gitlab.io/gitlab/coverage-javascript/), + [`webpack-report`](https://gitlab-org.gitlab.io/gitlab/webpack-report/). + +### Default image + +The default image is defined in [`.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml). + +It includes Ruby, Go, Git, Git LFS, Chrome, Node, Yarn, PostgreSQL, and Graphics Magick. + +The images used in our pipelines are configured in the +[`gitlab-org/gitlab-build-images`](https://gitlab.com/gitlab-org/gitlab-build-images) +project, which is push-mirrored to [`gitlab/gitlab-build-images`](https://dev.gitlab.org/gitlab/gitlab-build-images) +for redundancy. + +The current version of the build images can be found in the +["Used by GitLab section"](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml). + +### Default variables + +In addition to the [predefined variables](../ci/variables/predefined_variables.md), +each pipeline includes default variables defined in +<https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>. + +### Common job definitions + +Most of the jobs [extend from a few CI definitions](../ci/yaml/README.md#extends) +defined in [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml) +that are scoped to a single [configuration parameter](../ci/yaml/README.md#configuration-parameters). + +| Job definitions | Description | +|------------------|-------------| +| `.default-tags` | Ensures a job has the `gitlab-org` tag to ensure it's using our dedicated runners. | +| `.default-retry` | Allows a job to [retry](../ci/yaml/README.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. | +| `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (e.g. tests). | +| `.rails-cache` | Allows a job to use a default `cache` definition suitable for Ruby/Rails tasks. | +| `.static-analysis-cache` | Allows a job to use a default `cache` definition suitable for static analysis tasks. | +| `.yarn-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that do a `yarn install`. | +| `.assets-compile-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that compile assets. | +| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. | +| `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. | +| `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. | +| `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. | + +### `rules`, `if:` conditions and `changes:` patterns + +We're using the [`rules` keyword](../ci/yaml/README.md#rules) extensively. + +All `rules` definitions are defined in +<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>, +then included in individual jobs via [`extends`](../ci/yaml/README.md#extends). + +The `rules` definitions are composed of `if:` conditions and `changes:` patterns, +which are also defined in +[`rules.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml) +and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anchors) + +#### `if:` conditions + +| `if:` conditions | Description | Notes | +|------------------|-------------|-------| +| `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). | +| `if-not-ee` | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). | +| `if-not-foss` | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). | +| `if-default-refs` | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs won't be created for branches with this default configuration. | +| `if-master-refs` | Matches if the current branch is `master`. | | +| `if-master-or-tag` | Matches if the pipeline is for the `master` branch or for a tag. | | +| `if-merge-request` | Matches if the pipeline is for a merge request. | | +| `if-nightly-master-schedule` | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | | +| `if-dot-com-gitlab-org-schedule` | Limits jobs creation to scheduled pipelines for the `gitlab-org` group on GitLab.com. | | +| `if-dot-com-gitlab-org-master` | Limits jobs creation to the `master` branch for the `gitlab-org` group on GitLab.com. | | +| `if-dot-com-gitlab-org-merge-request` | Limits jobs creation to merge requests for the `gitlab-org` group on GitLab.com. | | +| `if-dot-com-gitlab-org-and-security-tag` | Limits job creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | +| `if-dot-com-gitlab-org-and-security-merge-request` | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | +| `if-dot-com-ee-schedule` | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | | +| `if-cache-credentials-schedule` | Limits jobs to scheduled pipelines with the `$CI_REPO_CACHE_CREDENTIALS` variable set. | | + +#### `changes:` patterns + +| `changes:` patterns | Description | +|------------------------------|--------------------------------------------------------------------------| +| `ci-patterns` | Only create job for CI config-related changes. | +| `yaml-patterns` | Only create job for YAML-related changes. | +| `docs-patterns` | Only create job for docs-related changes. | +| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. | +| `frontend-patterns` | Only create job for frontend-related changes. | +| `backstage-patterns` | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). | +| `code-patterns` | Only create job for code-related changes. | +| `qa-patterns` | Only create job for QA-related changes. | +| `code-backstage-patterns` | Combination of `code-patterns` and `backstage-patterns`. | +| `code-qa-patterns` | Combination of `code-patterns` and `qa-patterns`. | +| `code-backstage-qa-patterns` | Combination of `code-patterns`, `backstage-patterns`, and `qa-patterns`. | + --- [Return to Development documentation](README.md) diff --git a/doc/development/polling.md b/doc/development/polling.md index bc178f8cb21..47cfc32d934 100644 --- a/doc/development/polling.md +++ b/doc/development/polling.md @@ -56,4 +56,4 @@ For more information see: - [`Poll-Interval` header](fe_guide/performance.md#realtime-components) - [RFC 7232](https://tools.ietf.org/html/rfc7232) -- [ETag proposal](https://gitlab.com/gitlab-org/gitlab-foss/issues/26926) +- [ETag proposal](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/26926) diff --git a/doc/development/profiling.md b/doc/development/profiling.md index 2589329fc83..2cab6750b9b 100644 --- a/doc/development/profiling.md +++ b/doc/development/profiling.md @@ -107,9 +107,13 @@ Recorded transactions can be found by navigating to `/sherlock/transactions`. ## Bullet -Bullet is a Gem that can be used to track down N+1 query problems. Because -Bullet adds quite a bit of logging noise it's disabled by default. To enable -Bullet, set the environment variable `ENABLE_BULLET` to a non-empty value before +Bullet is a Gem that can be used to track down N+1 query problems. Bullet section is +displayed on the [performance-bar](../administration/monitoring/performance/performance_bar.md). + + + +Because Bullet adds quite a bit of logging noise the logging is disabled by default. +To enable the logging, set the environment variable `ENABLE_BULLET` to a non-empty value before starting GitLab. For example: ```shell diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index 96acce5e4df..3fa6ba40e6c 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -147,7 +147,7 @@ To run several tests inside one directory: ### Speed up tests, Rake tasks, and migrations -[Spring](https://github.com/rails/spring) is a Rails application preloader. It +[Spring](https://github.com/rails/spring) is a Rails application pre-loader. It speeds up development by keeping your application running in the background so you don't need to boot it every time you run a test, Rake task or migration. @@ -203,9 +203,9 @@ To generate a sprite file containing all the Emoji, run: bundle exec rake gemojione:sprite ``` -If new emoji are added, the spritesheet may change size. To compensate for -such changes, first generate the `emoji.png` spritesheet with the above Rake -task, then check the dimensions of the new spritesheet and update the +If new emoji are added, the sprite sheet may change size. To compensate for +such changes, first generate the `emoji.png` sprite sheet with the above Rake +task, then check the dimensions of the new sprite sheet and update the `SPRITESHEET_WIDTH` and `SPRITESHEET_HEIGHT` constants accordingly. ## Update project templates diff --git a/doc/development/redis.md b/doc/development/redis.md index a8b7b84bb65..6782ea96448 100644 --- a/doc/development/redis.md +++ b/doc/development/redis.md @@ -18,7 +18,7 @@ database. Redis is a flat namespace with no hierarchy, which means we must pay attention to key names to avoid collisions. Typically we use colon-separated elements to -provide a semblence of structure at application level. An example might be +provide a semblance of structure at application level. An example might be `projects:1:somekey`. Although we split our Redis usage into three separate purposes, and those may @@ -34,7 +34,7 @@ invalidated by a name change, it is better to include a hook that will expire the entry, instead of relying on the key changing. We don't use [Redis Cluster](https://redis.io/topics/cluster-tutorial) at the -moment, but may wish to in the future: [#118820](https://gitlab.com/gitlab-org/gitlab/issues/118820). +moment, but may wish to in the future: [#118820](https://gitlab.com/gitlab-org/gitlab/-/issues/118820). This imposes an additional constraint on naming: where GitLab is performing operations that require several keys to be held on the same Redis server - for diff --git a/doc/development/refactoring_guide/index.md b/doc/development/refactoring_guide/index.md index 4bd9d0e9c11..a9ff9556aed 100644 --- a/doc/development/refactoring_guide/index.md +++ b/doc/development/refactoring_guide/index.md @@ -21,7 +21,7 @@ Pinning tests help you ensure that you don't unintentionally change the output o Leaving in the commits for adding and removing pins helps others checkout and verify the result of the test. -```bash +```shell AAAAAA Add pinning tests to funky_foo BBBBBB Refactor funky_foo into nice_foo CCCCCC Remove pinning tests for funky_foo @@ -31,13 +31,13 @@ Then you can leave a reviewer instructions on how to run the pinning test in you > First revert the commit which removes the pin. > -> ```bash +> ```shell > git revert --no-commit $(git log -1 --grep="Remove pinning test for funky_foo" --pretty=format:"%H") > ``` > > Then run the test > -> ```bash +> ```shell > yarn run jest path/to/funky_foo_pin_spec.js > ``` @@ -69,7 +69,7 @@ expect(cleanForSnapshot(wrapper.element)).toMatchSnapshot(); ### Examples -- [Pinning test in a haml to vue refactor](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27691#pinning-tests) +- [Pinning test in a Haml to Vue refactor](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27691#pinning-tests) - [Pinning test in isolating a bug](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/32198#note_212736225) - [Pinning test in refactoring dropdown](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28173) - [Pinning test in refactoring vulnerability_details.vue](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25830/commits) diff --git a/doc/development/reference_processing.md b/doc/development/reference_processing.md index ef1f2f5269c..79377533966 100644 --- a/doc/development/reference_processing.md +++ b/doc/development/reference_processing.md @@ -27,7 +27,7 @@ transform them into structured links to the resources they represent. For example, the class [`Banzai::Filter::IssueReferenceFilter`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/banzai/filter/issue_reference_filter.rb) is responsible for handling references to issues, such as -`gitlab-org/gitlab#123` and `https://gitlab.com/gitlab-org/gitlab/issues/200048`. +`gitlab-org/gitlab#123` and `https://gitlab.com/gitlab-org/gitlab/-/issues/200048`. All reference filters are instances of [`HTML::Pipeline::Filter`](https://www.rubydoc.info/github/jch/html-pipeline/v1.11.0/HTML/Pipeline/Filter), and inherit (often indirectly) from [`Banzai::Filter::ReferenceFilter`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/banzai/filter/reference_filter.rb). diff --git a/doc/development/renaming_features.md b/doc/development/renaming_features.md index 6a196921a5d..daf437027db 100644 --- a/doc/development/renaming_features.md +++ b/doc/development/renaming_features.md @@ -16,7 +16,7 @@ The more of the following that are true, the more likely you should choose the f - You are not confident the new name is permanent. - The feature is susceptible to bugs (large, complex, needing refactor, etc). -- The renaming will be difficult to review (feature spans many lines/files/repos). +- The renaming will be difficult to review (feature spans many lines, files, or repositories). - The renaming will be disruptive in some way (database table renaming). ## Consider a façade-first approach diff --git a/doc/development/routing.md b/doc/development/routing.md index 97837a917a2..e164431853f 100644 --- a/doc/development/routing.md +++ b/doc/development/routing.md @@ -63,11 +63,19 @@ gitlab-org/gitlab/-/settings/repository gitlab-org/serverless/runtimes/-/settings/repository ``` -Currently, only some project routes are placed under the `/-/` scope. However, -you can help us migrate more of them! To migrate project routes: +## Migrating unscoped routes + +Currently, the majority of routes are placed under the `/-/` scope. However, +you can help us migrate the rest of them! To migrate routes: 1. Modify existing routes by adding `-` scope. 1. Add redirects for legacy routes by using `Gitlab::Routing.redirect_legacy_paths`. 1. Create a technical debt issue to remove deprecated routes in later releases. To get started, see an [example merge request](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28435). + +## Useful links + +- [Routing improvements master plan](https://gitlab.com/gitlab-org/gitlab/-/issues/215362) +- [Scoped routing explained](https://gitlab.com/gitlab-org/gitlab/-/issues/214217) +- [Removal of deprecated routes](https://gitlab.com/gitlab-org/gitlab/-/issues/28848) diff --git a/doc/development/scalability.md b/doc/development/scalability.md index ba25e169d66..c0c26df88b5 100644 --- a/doc/development/scalability.md +++ b/doc/development/scalability.md @@ -52,10 +52,10 @@ maintain and support one database with tables with many rows. There are two ways to deal with this: -- Partioning. Locally split up tables data. +- Partitioning. Locally split up tables data. - Sharding. Distribute data across multiple databases. -Partioning is a built-in PostgreSQL feature and requires minimal changes +Partitioning is a built-in PostgreSQL feature and requires minimal changes in the application. However, it [requires PostgreSQL 11](https://www.2ndquadrant.com/en/blog/partitioning-evolution-postgresql-11/). @@ -93,12 +93,12 @@ systems. #### Database size A recent [database checkup shows a breakdown of the table sizes on -GitLab.com](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/8022#master-1022016101-8). +GitLab.com](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/8022#master-1022016101-8). Since `merge_request_diff_files` contains over 1 TB of data, we will want to reduce/eliminate this table first. GitLab has support for [storing diffs in object storage](../administration/merge_request_diffs.md), which we [will want to do on -GitLab.com](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/7356). +GitLab.com](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7356). #### High availability @@ -116,7 +116,7 @@ database has reached the target time. On GitLab.com, Consul and Patroni work together to coordinate failovers with the read replicas. [Omnibus ships with repmgr instead of -Consul](../administration/high_availability/database.md). +Patroni](../administration/postgresql/replication_and_failover.md). #### Load-balancing @@ -147,10 +147,10 @@ limitation: - Run multiple PgBouncer instances. - Use a multi-threaded connection pooler (e.g. - [Odyssey](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/7776). + [Odyssey](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7776). On some Linux systems, it's possible to run [multiple PgBouncer instances on -the same port](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4796). +the same port](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4796). On GitLab.com, we run multiple PgBouncer instances on different ports to avoid saturating a single core. @@ -246,9 +246,9 @@ lifting of many activities, including: - Processing CI builds and pipelines. The full list of jobs can be found in the -[app/workers](https://gitlab.com/gitlab-org/gitlab/tree/master/app/workers) +[`app/workers`](https://gitlab.com/gitlab-org/gitlab/tree/master/app/workers) and -[ee/app/workers](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/workers) +[`ee/app/workers`](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/workers) directories in the GitLab code base. #### Runaway Queues @@ -275,13 +275,13 @@ in a timely manner: - Redistribute/gerrymander Sidekiq processes by queue types. Long-running jobs (e.g. relating to project import) can often squeeze out jobs that run fast (e.g. delivering e-mail). [This technique - was used in to optimize our existing Sidekiq deployment](https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/7219#note_218019483). + was used in to optimize our existing Sidekiq deployment](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7219#note_218019483). - Optimize jobs. Eliminating unnecessary work, reducing network calls (e.g. SQL, Gitaly, etc.), and optimizing processor time can yield significant benefits. From the Sidekiq logs, it's possible to see which jobs run the most -frequently and/or take the longest. For example, theis Kibana +frequently and/or take the longest. For example, these Kibana visualizations show the jobs that consume the most total time:  diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index b473c310647..912b8fbf043 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -23,7 +23,7 @@ For more information about the permission model at GitLab, please see [the GitLa ### Impact Improper permission handling can have significant impacts on the security of an application. -Some situations may reveal [sensitive data](https://gitlab.com/gitlab-com/gl-infra/production/issues/477) or allow a malicious actor to perform [harmful actions](https://gitlab.com/gitlab-org/gitlab/issues/8180). +Some situations may reveal [sensitive data](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/477) or allow a malicious actor to perform [harmful actions](https://gitlab.com/gitlab-org/gitlab/-/issues/8180). The overall impact depends heavily on what resources can be accessed or modified improperly. A common vulnerability when permission checks are missing is called [IDOR](https://owasp.org/www-project-web-security-testing-guide/latest/4-Web_Application_Security_Testing/05-Authorization_Testing/04-Testing_for_Insecure_Direct_Object_References) for Insecure Direct Object References. @@ -48,11 +48,11 @@ Be careful to **also test [visibility levels](https://gitlab.com/gitlab-org/gitl Some example of well implemented access controls and tests: -1. [example1](https://dev.gitlab.org/gitlab/gitlab-ee/merge_requests/710/diffs?diff_id=13750#af40ef0eaae3c1e018809e1d88086e32bccaca40_43_43) +1. [example1](https://dev.gitlab.org/gitlab/gitlab-ee/-/merge_requests/710/diffs?diff_id=13750#af40ef0eaae3c1e018809e1d88086e32bccaca40_43_43) 1. [example2](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/2511/diffs#ed3aaab1510f43b032ce345909a887e5b167e196_142_155) 1. [example3](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/3170/diffs?diff_id=17494) -**NB:** any input from development team is welcome, e.g. about rubocop rules. +**NB:** any input from development team is welcome, e.g. about Rubocop rules. ## Regular Expressions guidelines @@ -67,7 +67,7 @@ matches = re.findall("^bar$",text) print(matches) ``` -The Python example will output an emtpy array (`[]`) as the matcher considers the whole string `foo\nbar` including the newline (`\n`). In contrast Ruby's Regular Expression engine acts differently: +The Python example will output an empty array (`[]`) as the matcher considers the whole string `foo\nbar` including the newline (`\n`). In contrast Ruby's Regular Expression engine acts differently: ```ruby text = "foo\nbar" @@ -82,7 +82,7 @@ This Ruby Regex specialty can have security impact, as often regular expressions #### Examples -GitLab specific examples can be found [here](https://gitlab.com/gitlab-org/gitlab/issues/36029#note_251262187) and [there](https://gitlab.com/gitlab-org/gitlab/issues/33569). +GitLab specific examples can be found [here](https://gitlab.com/gitlab-org/gitlab/-/issues/36029#note_251262187) and [there](https://gitlab.com/gitlab-org/gitlab/-/issues/33569). Another example would be this fictional Ruby On Rails controller: @@ -111,7 +111,7 @@ or controls the regular expression (regex) used, and is able to enter user input ### Impact -The resource, for example Unicorn, Puma, or Sidekiq, can be made to hang as it takes a long time to evaulate the bad regex match. +The resource, for example Unicorn, Puma, or Sidekiq, can be made to hang as it takes a long time to evaluate the bad regex match. ### Examples @@ -140,9 +140,9 @@ class Email < ApplicationRecord GitLab has `Gitlab::UntrustedRegexp` which internally uses the [`re2`](https://github.com/google/re2/wiki/Syntax) library. By utilizing `re2`, we get a strict limit on total execution time, and a smaller subset of available regex features. -All user-provided regexes should use `Gitlab::UntrustedRegexp`. +All user-provided regular expressions should use `Gitlab::UntrustedRegexp`. -For other regexes, here are a few guidelines: +For other regular expressions, here are a few guidelines: - Remove unnecessary backtracking. - Avoid nested quantifiers if possible. @@ -180,11 +180,11 @@ have been reported to GitLab include: - Network mapping of internal services - This can help an attacker gather information about internal services - that could be used in further attacks. [More details](https://gitlab.com/gitlab-org/gitlab-foss/issues/51327). + that could be used in further attacks. [More details](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51327). - Reading internal services, including cloud service metadata. - The latter can be a serious problem, because an attacker can obtain keys that allow control of the victim's cloud infrastructure. (This is also a good reason - to give only necessary privileges to the token.). [More details](https://gitlab.com/gitlab-org/gitlab-foss/issues/51490). -- When combined with CRLF vulnerability, remote code execution. [More details](https://gitlab.com/gitlab-org/gitlab-foss/issues/41293) + to give only necessary privileges to the token.). [More details](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51490). +- When combined with CRLF vulnerability, remote code execution. [More details](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41293) ### When to Consider @@ -206,14 +206,14 @@ The [GitLab::HTTP](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab `Outbound requests` options that allow instance administrators to block all internal connections, or limit the networks to which connections can be made. In some cases, it has been possible to configure GitLab::HTTP as the HTTP -connection library for 3rd-party gems. This is preferrable over re-implementing +connection library for 3rd-party gems. This is preferable over re-implementing the mitigations for a new feature. - [More details](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/2530/diffs) #### Feature-specific Mitigations -For situtions in which a whitelist or GitLab:HTTP cannot be used, it will be necessary to implement mitigations directly in the feature. It is best to validate the destination IP addresses themselves, not just domain names, as DNS can be controlled by the attacker. Below are a list of mitigations that should be implemented. +For situations in which an allowlist or GitLab:HTTP cannot be used, it will be necessary to implement mitigations directly in the feature. It is best to validate the destination IP addresses themselves, not just domain names, as DNS can be controlled by the attacker. Below are a list of mitigations that should be implemented. **Important Note:** There are many tricks to bypass common SSRF validations. If feature-specific mitigations are necessary, they should be reviewed by the AppSec team, or a developer who has worked on SSRF mitigations previously. @@ -230,7 +230,7 @@ For situtions in which a whitelist or GitLab:HTTP cannot be used, it will be nec - For HTTP connections: Disable redirects or validate the redirect destination - To mitigate DNS rebinding attacks, validate and use the first IP address received -See [url_blocker_spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/lib/gitlab/url_blocker_spec.rb) for examples of SSRF payloads +See [`url_blocker_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/lib/gitlab/url_blocker_spec.rb) for examples of SSRF payloads ## XSS guidelines @@ -276,10 +276,10 @@ For any and all input fields, ensure to define expectations on the type/format o - Treat all user input as untrusted. - Based on the expectations you [defined above](#setting-expectations): - Validate the [input size limits](https://youtu.be/2VFavqfDS6w?t=7582). - - Validate the input using a [whitelist approach](https://youtu.be/2VFavqfDS6w?t=7816) to only allow characters through which you are expecting to receive for the field. + - Validate the input using an [allowlist approach](https://youtu.be/2VFavqfDS6w?t=7816) to only allow characters through which you are expecting to receive for the field. - Input which fails validation should be **rejected**, and not sanitized. -Note that blacklists should be avoided, as it is near impossible to block all [variations of XSS](https://owasp.org/www-community/xss-filter-evasion-cheatsheet). +Note that denylists should be avoided, as it is near impossible to block all [variations of XSS](https://owasp.org/www-community/xss-filter-evasion-cheatsheet). #### Output encoding @@ -308,7 +308,7 @@ Once you've [determined when and where](#setting-expectations) the user submitte #### Content Security Policy - [Content Security Policy](https://www.youtube.com/watch?v=2VFavqfDS6w&t=12991s) -- [Use nonce-based Content Security Policy for inline JavaScript](https://gitlab.com/gitlab-org/gitlab-foss/issues/65330) +- [Use nonce-based Content Security Policy for inline JavaScript](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/65330) #### Free form input fields @@ -323,7 +323,7 @@ Once you've [determined when and where](#setting-expectations) the user submitte ### Select examples of past XSS issues affecting GitLab -- [Stored XSS in user status](https://gitlab.com/gitlab-org/gitlab-foss/issues/55320) +- [Stored XSS in user status](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55320) ### Developer Training @@ -345,5 +345,5 @@ Once you've [determined when and where](#setting-expectations) the user submitte - [Input Validation](https://youtu.be/2VFavqfDS6w?t=7489) - [Validate size limits](https://youtu.be/2VFavqfDS6w?t=7582) - [RoR model validators](https://youtu.be/2VFavqfDS6w?t=7636) -- [Whitelist input validation](https://youtu.be/2VFavqfDS6w?t=7816) +- [Allowlist input validation](https://youtu.be/2VFavqfDS6w?t=7816) - [Content Security Policy](https://www.youtube.com/watch?v=2VFavqfDS6w&t=12991s) diff --git a/doc/development/service_measurement.md b/doc/development/service_measurement.md new file mode 100644 index 00000000000..e53864c8640 --- /dev/null +++ b/doc/development/service_measurement.md @@ -0,0 +1,81 @@ +# GitLab Developers Guide to service measurement + +You can enable service measurement in order to debug any slow service's execution time, number of SQL calls, garbage collection stats, memory usage, etc. + +## Measuring module + +The measuring module is a tool that allows to measure a service's execution, and log: + +- Service class name +- Execution time +- Number of SQL calls +- Detailed `gc` stats and diffs +- RSS memory usage +- Server worker ID + +The measuring module will log these measurements into a structured log called [`service_measurement.log`](../administration/logs.md#service_measurementlog), +as a single entry for each service execution. + +NOTE: **Note:** +For GitLab.com, `service_measurement.log` is ingested in Elasticsearch and Kibana as part of our monitoring solution. + +## How to use it + +The measuring module allows you to easily measure and log execution of any service, +by just prepending `Measurable` in any Service class, on the last line of the file that the class resides in. + +For example, to prepend a module into the `DummyService` class, you would use the following approach: + +```ruby +class DummyService + def execute + # ... + end +end + +DummyService.prepend(Measurable) +``` + +In case when you are prepending a module from the `EE` namespace with EE features, you need to prepend Measurable after prepending the `EE` module. + +This way, `Measurable` will be at the bottom of the ancestor chain, in order to measure execution of `EE` features as well: + +```ruby +class DummyService + def execute + # ... + end +end + +DummyService.prepend_if_ee('EE::DummyService') +DummyService.prepend(Measurable) +``` + +### Log additional attributes + +In case you need to log some additional attributes, it is possible to define `extra_attributes_for_measurement` in the service class: + +```ruby +def extra_attributes_for_measurement + { + project_path: @project.full_path, + user: current_user.name + } +end +``` + +NOTE: **Note:** +Once the measurement module is injected in the service, it will be behind generic feature flag. +In order to actually use it, you need to enable measuring for the desired service by enabling the feature flag. + +### Enabling measurement using feature flags + +In the following example, the `:gitlab_service_measuring_projects_import_service` +[feature flag](feature_flags/development.md#enabling-a-feature-flag-in-development) is used to enable the measuring feature +for `Projects::ImportService`. + +From ChatOps: + +```shell +/chatops run feature set gitlab_service_measuring_projects_import_service true +``` diff --git a/doc/development/shell_scripting_guide/index.md b/doc/development/shell_scripting_guide/index.md index 99cd1b9d67f..c04a4e90e59 100644 --- a/doc/development/shell_scripting_guide/index.md +++ b/doc/development/shell_scripting_guide/index.md @@ -106,7 +106,7 @@ and ignore files starting with a period. To override this, use `-ln` flag to spe NOTE: **Note:** This is a work in progress. -It is an [ongoing effort](https://gitlab.com/gitlab-org/gitlab-foss/issues/64016) to evaluate different tools for the +It is an [ongoing effort](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64016) to evaluate different tools for the automated testing of shell scripts (like [BATS](https://github.com/bats-core/bats-core)). ## Code Review diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md index a5d0eecdc7b..7ae3c9e9de2 100644 --- a/doc/development/sidekiq_style_guide.md +++ b/doc/development/sidekiq_style_guide.md @@ -78,7 +78,7 @@ As a general rule, a worker can be considered idempotent if: - It can safely run multiple times with the same arguments. - Application side-effects are expected to happen only once - (or side-effects of a second run are not impactful). + (or side-effects of a second run do not have an effect). A good example of that would be a cache expiration worker. @@ -147,7 +147,7 @@ GitLab doesn't skip jobs scheduled in the future, as we assume that the state will have changed by the time the job is scheduled to execute. -More [deduplication strategies have been suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/195). If you are implementing a worker that +More [deduplication strategies have been suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/195). If you are implementing a worker that could benefit from a different strategy, please comment in the issue. If the automatic deduplication were to cause issues in certain @@ -156,7 +156,7 @@ named `disable_<queue name>_deduplication`. For example to disable deduplication for the `AuthorizedProjectsWorker`, we would enable the feature flag `disable_authorized_projects_deduplication`. -From chatops: +From ChatOps: ```shell /chatops run feature set disable_authorized_projects_deduplication true @@ -272,10 +272,10 @@ annotated with the `worker_resource_boundary` method. Most workers tend to spend most of their time blocked, wait on network responses from other services such as Redis, PostgreSQL, and Gitaly. Since Sidekiq is a -multithreaded environment, these jobs can be scheduled with high concurrency. +multi-threaded environment, these jobs can be scheduled with high concurrency. Some workers, however, spend large amounts of time _on-CPU_ running logic in -Ruby. Ruby MRI does not support true multithreading - it relies on the +Ruby. Ruby MRI does not support true multi-threading - it relies on the [GIL](https://thoughtbot.com/blog/untangling-ruby-threads#the-global-interpreter-lock) to greatly simplify application development by only allowing one section of Ruby code in a process to run at a time, no matter how many cores the machine @@ -395,7 +395,7 @@ in the default execution mode - using does not account for weights. As we are [moving towards using `sidekiq-cluster` in -Core](https://gitlab.com/gitlab-org/gitlab/issues/34396), newly-added +Core](https://gitlab.com/gitlab-org/gitlab/-/issues/34396), newly-added workers do not need to have weights specified. They can simply use the default weight, which is 1. @@ -427,7 +427,7 @@ isn't picked up by the cops. In any case, please leave a code-comment pointing to which context will be used when disabling the cops. When you do provide objects to the context, please make sure that the -route for namespaces and projects is preloaded. This can be done using +route for namespaces and projects is pre-loaded. This can be done using the `.with_route` scope defined on all `Routable`s. ### Cron-Workers @@ -518,6 +518,34 @@ job needs to be scheduled with. The `context_proc` which needs to return a hash with the context information for the job. +## Arguments logging + +When [`SIDEKIQ_LOG_ARGUMENTS`](../administration/troubleshooting/sidekiq.md#log-arguments-to-sidekiq-jobs) +is enabled, Sidekiq job arguments will be logged. + +By default, the only arguments logged are numeric arguments, because +arguments of other types could contain sensitive information. To +override this, use `loggable_arguments` inside a worker with the indexes +of the arguments to be logged. (Numeric arguments do not need to be +specified here.) + +For example: + +```ruby +class MyWorker + include ApplicationWorker + + loggable_arguments 1, 3 + + # object_id will be logged as it's numeric + # string_a will be logged due to the loggable_arguments call + # string_b will be filtered from logs + # string_c will be logged due to the loggable_arguments call + def perform(object_id, string_a, string_b, string_c) + end +end +``` + ## Tests Each Sidekiq worker must be tested using RSpec, just like any other class. These @@ -537,18 +565,74 @@ possible situations: ### Changing the arguments for a worker -Jobs need to be backwards- and forwards-compatible between consecutive versions -of the application. +Jobs need to be backward and forward compatible between consecutive versions +of the application. Adding or removing an argument may cause problems +during deployment before all Rails and Sidekiq nodes have the updated code. + +#### Remove an argument + +**Do not remove arguments from the `perform` function.**. Instead, use the +following approach: + +1. Provide a default value (usually `nil`) and use a comment to mark the + argument as deprecated +1. Stop using the argument in `perform_async`. +1. Ignore the value in the worker class, but do not remove it until the next + major release. + +In the following example, if you want to remove `arg2`, first set a `nil` default value, +and then update locations where `ExampleWorker.perform_async` is called. + +```ruby +class ExampleWorker + def perform(object_id, arg1, arg2 = nil) + # ... + end +end +``` + +#### Add an argument + +There are two options for safely adding new arguments to Sidekiq workers: + +1. Set up a [multi-step deployment](#multi-step-deployment) in which the new argument is first added to the worker +1. Use a [parameter hash](#parameter-hash) for additional arguments. This is perhaps the most flexible option. +1. Use a parameter hash for additional arguments. This is perhaps the most flexible option. + +##### Multi-step deployment + +This approach requires multiple merge requests and for the first merge request +to be merged and deployed before additional changes are merged. -This can be done by following this process: +1. In an initial merge request, add the argument to the worker with a default + value: -1. **Do not remove arguments from the `perform` function.**. Instead, use the - following approach - 1. Provide a default value (usually `nil`) and use a comment to mark the - argument as deprecated - 1. Stop using the argument in `perform_async`. - 1. Ignore the value in the worker class, but do not remove it until the next - major release. + ```ruby + class ExampleWorker + def perform(object_id, new_arg = nil) + # ... + end + end + ``` + +1. Merge and deploy the worker with the new argument. +1. In a further merge request, update `ExampleWorker.perform_async` calls to + use the new argument. + +##### Parameter hash + +This approach will not require multiple deployments if an existing worker already +utilizes a parameter hash. + +1. Use a parameter hash in the worker to allow for future flexibility: + + ```ruby + class ExampleWorker + def perform(object_id, params = {}) + # ... + end + end + ``` ### Removing workers diff --git a/doc/development/telemetry/index.md b/doc/development/telemetry/index.md index 32f63d5221e..aee16e4049a 100644 --- a/doc/development/telemetry/index.md +++ b/doc/development/telemetry/index.md @@ -1,3 +1,9 @@ +--- +stage: Growth +group: Telemetry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Telemetry Guide At GitLab, we collect telemetry for the purpose of helping us build a better GitLab. Data about how GitLab is used is collected to better understand what parts of GitLab needs improvement and what features to build next. Telemetry also helps our team better understand the reasons why people use GitLab and with this knowledge we are able to make better product decisions. @@ -19,11 +25,11 @@ Telemetry Guide: 1. [What is Usage Ping](usage_ping.md#what-is-usage-ping) 1. [Usage Ping payload](usage_ping.md#usage-ping-payload) - 1. [Disabling Usage Ping](usage_ping.md#disabling-usage-ping) + 1. [Disable Usage Ping](usage_ping.md#disable-usage-ping) 1. [Usage Ping request flow](usage_ping.md#usage-ping-request-flow) 1. [How Usage Ping works](usage_ping.md#how-usage-ping-works) 1. [Implementing Usage Ping](usage_ping.md#implementing-usage-ping) - 1. [Developing and testing usage ping](usage_ping.md#developing-and-testing-usage-ping) + 1. [Developing and testing Usage Ping](usage_ping.md#developing-and-testing-usage-ping) [Snowplow Guide](snowplow.md) @@ -44,27 +50,27 @@ More useful links: ## Our tracking tools -In this section we will explain the six different technologies we use to gather product usage data. +We use several different technologies to gather product usage data. -**Snowplow JS (Frontend)** +### Snowplow JS (Frontend) Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. [Snowplow JS](https://github.com/snowplow/snowplow/wiki/javascript-tracker) is a frontend tracker for client-side events. -**Snowplow Ruby (Backend)** +### Snowplow Ruby (Backend) Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. [Snowplow Ruby](https://github.com/snowplow/snowplow/wiki/ruby-tracker) is a backend tracker for server-side events. -**Usage Ping** +### Usage Ping Usage Ping is a method for GitLab Inc to collect usage data on a GitLab instance. Usage Ping is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. This high-level data is used to help our product, support, and sales teams. -Read more about how this works in the [Usage Ping guide](usage_ping.md) +For more details, read the [Usage Ping](usage_ping.md) guide. -**Database import** +### Database import Database imports are full imports of data into GitLab's data warehouse. For GitLab.com, the PostgreSQL database is loaded into Snowflake data warehouse every 6 hours. For more details, see the [data team handbook](https://about.gitlab.com/handbook/business-ops/data-team/#extract-and-load). -**Log system** +### Log system System logs are the application logs generated from running the GitLab Rails application. For more details, see the [log system](../../administration/logs.md) and [logging infrastructure](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#logging-infrastructure-overview). @@ -72,63 +78,63 @@ System logs are the application logs generated from running the GitLab Rails app Our different tracking tools allows us to track different types of events. The event types and examples of what data can be tracked are outlined below. -| Event Type | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Log system | -| ------ | ------ | ------ | ------ | ------ | ------ | -| Database counts | ❌ | ❌ | ✅ | ✅ | ❌ | -| Pageview events | ✅ | ✅ | ❌ | ❌ | ❌ | -| UI events | ✅ | ❌ | ❌ | ❌ | ❌ | -| CRUD and API events | ❌ | ✅ | ❌ | ❌ | ❌ | -| Event funnels | ✅ | ✅ | ❌ | ❌ | ❌ | -| PostgreSQL Data | ❌ | ❌ | ❌ | ✅ | ❌ | -| Logs | ❌ | ❌ | ❌ | ❌ | ✅ | -| External services | ❌ | ❌ | ❌ | ❌ | ❌ | +| Event Type | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Log system | +|---------------------|------------------------|-------------------------|---------------------|---------------------|---------------------| +| Database counts | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | +| Pageview events | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| UI events | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| CRUD and API events | **{dotted-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| Event funnels | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| PostgreSQL Data | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | **{dotted-circle}** | +| Logs | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | +| External services | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | **{dotted-circle}** | -**Database counts** +### Database counts -- How many Projects have been created by unique users -- How many users logged in the past 28 day +- Number of Projects created by unique users +- Number of users logged in the past 28 day -Database counts are row counts for different tables in an instance’s database. These are SQL count queries which have been filtered, grouped, or aggregated which provide high level usage data. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql) +Database counts are row counts for different tables in an instance’s database. These are SQL count queries which have been filtered, grouped, or aggregated which provide high level usage data. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). -**Pageview events** +### Pageview events -- How many sessions visited the /dashboard/groups page +- Number of sessions that visited the /dashboard/groups page -**UI Events** +### UI Events -- How many sessions clicked on a button or link -- How many sessions closed a modal +- Number of sessions that clicked on a button or link +- Number of sessions that closed a modal UI events are any interface-driven actions from the browser including click data. -**CRUD or API events** +### CRUD or API events -- How many Git pushes were made -- How many GraphQL queries were made -- How many requests were made to a Rails action or controller. +- Number of Git pushes +- Number of GraphQL queries +- Number of requests to a Rails action or controller -These are backend events that include the creation, read, update, deletion of records and other events that might be triggered from layers that aren't necessarily only available in the interface. +These are backend events that include the creation, read, update, deletion of records, and other events that might be triggered from layers other than those available in the interface. -**Event funnels** +### Event funnels -- How many sessions performed action A, B, then C -- What is our conversion rate from step A to B? +- Number of sessions that performed action A, B, then C +- Conversion rate from step A to B -**PostgreSQL data** +### PostgreSQL data -These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql) +These are raw database records which can be explored using business intelligence tools like Sisense. The full list of available tables can be found in [structure.sql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/structure.sql). -**Logs** +### Logs These are raw logs such as the [Production logs](../../administration/logs.md#production_jsonlog), [API logs](../../administration/logs.md#api_jsonlog), or [Sidekiq logs](../../administration/logs.md#sidekiqlog). See the [overview of Logging Infrastructure](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#logging-infrastructure-overview) for more details. -**External services** +### External services These are external services a GitLab instance interacts with such as an [external storage provider](../../administration/static_objects_external_storage.md) or an [external container registry](../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint). These services must be able to send data back into a GitLab instance for data to be tracked. ## Telemetry systems overview -The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed nstances. +The systems overview is a simplified diagram showing the interactions between GitLab Inc and self-managed instances.  @@ -140,7 +146,7 @@ For Telemetry purposes, GitLab Inc has three major components: 1. [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/): This contains everything managed by our data team including Sisense Dashboards for visualization, Snowflake for Data Warehousing, incoming data sources such as PostgreSQL Pipeline and S3 Bucket, and lastly our data collectors [GitLab.com's Snowplow Collector](https://about.gitlab.com/handbook/engineering/infrastructure/library/snowplow/) and GitLab's Versions Application. 1. GitLab.com: This is the production GitLab application which is made up of a Client and Server. On the Client or browser side, a Snowplow JS Tracker (Frontend) is used to track client-side events. On the Server or application side, a Snowplow Ruby Tracker (Backend) is used to track server-side events. The server also contains Usage Ping which leverages a PostgreSQL database and a Redis in-memory data store to report on usage data. Lastly, the server also contains System Logs which are generated from running the GitLab application. -1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana +1. [Monitoring infrastructure](https://about.gitlab.com/handbook/engineering/monitoring/): This is the infrastructure used to ensure GitLab.com is operating smoothly. System Logs are sent from GitLab.com to our monitoring infrastructure and collected by a FluentD collector. From FluentD, logs are either sent to long term Google Cloud Services cold storage via Stackdriver, or, they are sent to our Elastic Cluster via Cloud Pub/Sub which can be explored in real-time using Kibana. ### Self-managed @@ -151,15 +157,15 @@ For Telemetry purposes, self-managed instances have two major components: ### Differences between GitLab Inc and Self-managed -As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Ping, and PostgreSQL database imports all flow into GitLab Inc's data fnfrastructure. However, on self-managed, only Usage Ping flows into GitLab Inc's data infrastructure. +As shown by the orange lines, on GitLab.com Snowplow JS, Snowplow Ruby, Usage Ping, and PostgreSQL database imports all flow into GitLab Inc's data infrastructure. However, on self-managed, only Usage Ping flows into GitLab Inc's data infrastructure. As shown by the green lines, on GitLab.com system logs flow into GitLab Inc's monitoring infrastructure. On self-managed, there are no logs sent to GitLab Inc's monitoring infrastructure. The differences between GitLab.com and self-managed are summarized below: -| Environment | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Logs system | -| ------ | ------ | ------ | ------ | ------ | ------ | -| GitLab.com | ✅ | ✅ | ✅ | ✅ | ✅ | -| Self-Managed | ❌(1) | ❌(1) | ✅ | ❌ | ❌ | +| Environment | Snowplow JS (Frontend) | Snowplow Ruby (Backend) | Usage Ping | Database import | Logs system | +|--------------|------------------------|-------------------------|--------------------|---------------------|---------------------| +| GitLab.com | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Self-Managed | **{dotted-circle}**(1) | **{dotted-circle}**(1) | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | Note (1): Snowplow JS and Snowplow Ruby are available on self-managed, however, the Snowplow Collector endpoint is set to a self-managed Snowplow Collector which GitLab Inc does not have access to. diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md index aeaad6e5624..b7090ee4d20 100644 --- a/doc/development/telemetry/snowplow.md +++ b/doc/development/telemetry/snowplow.md @@ -1,3 +1,9 @@ +--- +stage: Growth +group: Telemetry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Snowplow Guide This guide provides a details about how Snowplow works. It includes the following sections: @@ -40,7 +46,7 @@ From [Snowplow's documentation](https://github.com/snowplow/snowplow), Snowplow ## Snowplow schema -We currently have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/issues/207930) including the following definitions: +We currently have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: - Frontend and backend taxonomy as listed below - [Feature instrumentation taxonomy](https://about.gitlab.com/handbook/product/feature-instrumentation/#taxonomy) @@ -121,7 +127,7 @@ Below is an example of `data-track-*` attributes assigned to a button: /> ``` -Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows for them to be properly handled on rerendering and changes to the DOM, but it's important to know that because of the way these events are bound, click events shouldn't be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you'll need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). +Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows for them to be properly handled on re-rendering and changes to the DOM, but it's important to know that because of the way these events are bound, click events shouldn't be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you'll need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). Below is a list of supported `data-track-*` attributes: @@ -213,7 +219,7 @@ button.addEventListener('click', () => { ### Tests and test helpers -In Jest particularly in vue tests, you can use the following: +In Jest particularly in Vue tests, you can use the following: ```javascript import { mockTracking } from 'helpers/tracking_helper'; @@ -330,10 +336,10 @@ Snowplow Inspector Chrome Extension is a browser extension for testing frontend Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. -Snowplow Micro is a docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. +Snowplow Micro is a Docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. - Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) -- Look at the [Snowplow Micro repo](https://github.com/snowplow-incubator/snowplow-micro) +- Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) - Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) 1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro) diff --git a/doc/development/telemetry/usage_ping.md b/doc/development/telemetry/usage_ping.md index e9b959eaa96..0f438e02772 100644 --- a/doc/development/telemetry/usage_ping.md +++ b/doc/development/telemetry/usage_ping.md @@ -1,19 +1,17 @@ -# Usage Ping Guide +--- +stage: Growth +group: Telemetry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -> - [Introduced][ee-557] in GitLab Enterprise Edition 8.10. -> - More statistics [were added][ee-735] in GitLab Enterprise Edition 8.12. -> - [Moved to GitLab Core][ce-23361] in 9.1. -> - More statistics [were added][ee-6602] in GitLab Ultimate 11.2. +# Usage Ping Guide -This guide provides a details about how usage ping works. It includes the following sections: +> - Introduced in GitLab Enterprise Edition 8.10. +> - More statistics were added in GitLab Enterprise Edition 8.12. +> - Moved to GitLab Core in 9.1. +> - More statistics were added in GitLab Ultimate 11.2. -1. [What is Usage Ping](#what-is-usage-ping) -1. [Usage Ping payload](#usage-ping-payload) -1. [Disabling Usage Ping](#disabling-usage-ping) -1. [Usage Ping request flow](#usage-ping-request-flow) -1. [How Usage Ping works](#how-usage-ping-works) -1. [Implementing Usage Ping](#implementing-usage-ping) -1. [Developing and testing usage ping](#developing-and-testing-usage-ping) +This guide describes Usage Ping's purpose and how it's implemented. For more information about Telemetry, see: @@ -27,237 +25,50 @@ More useful links: - [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/data-for-product-managers/) - [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/data-infrastructure/) -## What is Usage Ping +## What is Usage Ping? -- GitLab sends a weekly payload containing usage data to GitLab Inc. The usage ping uses high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. -- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. -- Usage ping is important to GitLab as we use it to calculate our and Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. +- GitLab sends a weekly payload containing usage data to GitLab Inc. Usage Ping provides high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. +- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. In addition to counts, other facts + that help us classify and understand GitLab installations are collected. +- Usage ping is important to GitLab as we use it to calculate our Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. - Once usage ping is enabled, GitLab will gather data from the other instances and will be able to show usage statistics of your instance to your users. -### Why Should We Enable Usage Ping? +### Why should we enable Usage Ping? -- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we are able to make better product decisions. +- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we're able to make better product decisions. - As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. - As a benefit of having the usage ping active, GitLab provides you with The DevOps Score,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. - You will get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) - You will get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? - You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. +- Usage Ping is enabled by default. To disable it, see [Disable Usage Ping](#disable-usage-ping). ### Limitations -- Usage Ping does not track frontend events things like page views, link clicks, or user sessions and only focuses on aggregated backend events. +- Usage Ping does not track frontend events things like page views, link clicks, or user sessions, and only focuses on aggregated backend events. - Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. ## Usage Ping payload You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: -1. Navigate to the **Admin Area > Settings > Metrics and profiling**. +1. Navigate to **Admin Area > Settings > Metrics and profiling**. 1. Expand the **Usage statistics** section. 1. Click the **Preview payload** button. -Here is an example of the payload structure - -``` json -{ - "uuid": "0000000-0000-0000-0000-000000000000", - "hostname": "example.com", - "version": "12.10.0-pre", - "installation_type": "omnibus-gitlab", - "active_user_count": 999, - "recorded_at": "2020-04-17T07:43:54.162+00:00", - "edition": "EEU", - "license_md5": "00000000000000000000000000000000", - "license_id": null, - "historical_max_users": 999, - "licensee": { - "Name": "ABC, Inc.", - "Email": "email@example.com", - "Company": "ABC, Inc." - }, - "license_user_count": 999, - "license_starts_at": "2020-01-01", - "license_expires_at": "2021-01-01", - "license_plan": "ultimate", - "license_add_ons": { - }, - "license_trial": false, - "counts": { - "assignee_lists": 999, - "boards": 999, - "ci_builds": 999, - ... - }, - "container_registry_enabled": true, - "dependency_proxy_enabled": false, - "gitlab_shared_runners_enabled": true, - "gravatar_enabled": true, - "influxdb_metrics_enabled": true, - "ldap_enabled": false, - "mattermost_enabled": false, - "omniauth_enabled": true, - "prometheus_metrics_enabled": false, - "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", - "signup_enabled": true, - "web_ide_clientside_preview_enabled": true, - "ingress_modsecurity_enabled": true, - "projects_with_expiration_policy_disabled": 999, - "projects_with_expiration_policy_enabled": 999, - ... - "elasticsearch_enabled": true, - "license_trial_ends_on": null, - "geo_enabled": false, - "git": { - "version": { - "major": 2, - "minor": 26, - "patch": 1 - } - }, - "gitaly": { - "version": "12.10.0-rc1-93-g40980d40", - "servers": 56, - "filesystems": [ - "EXT_2_3_4" - ] - }, - "gitlab_pages": { - "enabled": true, - "version": "1.17.0" - }, - "database": { - "adapter": "postgresql", - "version": "9.6.15" - }, - "app_server": { - "type": "console" - }, - "avg_cycle_analytics": { - "issue": { - "average": 999, - "sd": 999, - "missing": 999 - }, - "plan": { - "average": null, - "sd": 999, - "missing": 999 - }, - "code": { - "average": null, - "sd": 999, - "missing": 999 - }, - "test": { - "average": null, - "sd": 999, - "missing": 999 - }, - "review": { - "average": null, - "sd": 999, - "missing": 999 - }, - "staging": { - "average": null, - "sd": 999, - "missing": 999 - }, - "production": { - "average": null, - "sd": 999, - "missing": 999 - }, - "total": 999 - }, - "usage_activity_by_stage": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - }, - "usage_activity_by_stage_monthly": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - } -} -``` +For an example payload, see [Example Usage Ping payload](#example-usage-ping-payload). -## Disabling usage ping +## Disable Usage Ping -The usage ping is opt-out. If you want to deactivate this feature, go to the Settings page of your administration panel and uncheck the Usage Ping checkbox. +To disable Usage Ping in the GitLab UI, go to the **Settings** page of your administration panel and uncheck the **Usage Ping** checkbox. -To disable the usage ping and prevent it from being configured in future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): +To disable Usage Ping and prevent it from being configured in the future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): ```ruby gitlab_rails['usage_ping_enabled'] = false ``` -And source installs can set the following in `gitlab.yml`: +Source installations can set the following in `gitlab.yml`: ```yaml production: &base @@ -267,9 +78,9 @@ production: &base usage_ping_enabled: false ``` -## Usage Ping Request Flow +## Usage Ping request flow -The following example shows a basic request/response flow between a GitLab Instance, the Versions Application, the License Application, Salesforce, GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense.: +The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, GitLab's S3 Bucket, GitLab's Snowflake Data Warehouse, and Sisense: ```mermaid sequenceDiagram @@ -303,34 +114,40 @@ sequenceDiagram ## How Usage Ping works 1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. -1. When the cron job runs, it calls [GitLab::UsageData.to_json](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). -1. GitLab::UsageData.to_json [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. -1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in GitLab::UsageData.to_json. +1. When the cron job runs, it calls [`GitLab::UsageData.to_json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). +1. `GitLab::UsageData.to_json` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. +1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in `GitLab::UsageData.to_json`. 1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20). ## Implementing Usage Ping -Usage Ping consists of four types of counters which are all found in `usage_data.rb`: +Usage Ping consists of two kinds of data, counters and observations. Counters track how often a certain event +happened over time, such as how many CI pipelines have run. They are monotonic and always trend up. +Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no +general guidelines around how to collect those, due to the individual nature of that data. + +There are four types of counters which are all found in `usage_data.rb`: - **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation - **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column - **Alternative Counters:** Used for settings and configurations - **Redis Counters:** Used for in-memory counts. This method is being deprecated due to data inaccuracies and will be replaced with a persistent method. -Note: Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. +NOTE: **Note:** +Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. ### Why batch counting For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. -For GitLab.com, there are extremely large tables with 15 second query timeouts, so, we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: +For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: -| Table | Row counts in millions | -| ------ | ------ | -| merge_request_diff_commits | 2280 | -| ci_build_trace_sections | 1764 | -| merge_request_diff_files | 1082 | -| events | 514 | +| Table | Row counts in millions | +|------------------------------|------------------------| +| `merge_request_diff_commits` | 2280 | +| `ci_build_trace_sections` | 1764 | +| `merge_request_diff_files` | 1082 | +| `events` | 514 | There are two batch counting methods provided, `Ordinary Batch Counters` and `Distinct Batch Counters`. Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, a specialized index may need to be added on the columns involved in a counter. @@ -393,7 +210,7 @@ Method: `redis_usage_data(counter, &block)` Arguments: - `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented -- or a `block`: wich is evaluated +- or a `block`: which is evaluated Example of usage: @@ -413,8 +230,8 @@ Method: `alt_usage_data(value = nil, fallback: -1, &block)` Arguments: -- `value`: a simple static value in wich case the value is simply returned. -- or a `block`: wich is evaluated +- `value`: a simple static value in which case the value is simply returned. +- or a `block`: which is evaluated - `fallback: -1`: the common value used for any metrics that are failing. Example of usage: @@ -425,6 +242,29 @@ alt_usage_data { Gitlab::CurrentSettings.uuid } alt_usage_data(999) ``` +### Prometheus Queries + +In those cases where operational metrics should be part of Usage Ping, a database or Redis query is unlikely +to provide useful data. Instead, Prometheus might be more appropriate, since most of GitLab's architectural +components publish metrics to it that can be queried back, aggregated, and included as usage data. + +NOTE: **Note:** +Prometheus as a data source for Usage Ping is currently only available for single-node Omnibus installations +that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance. + +In order to query Prometheus for metrics, a helper method is available that will `yield` a fully configured +`PrometheusClient`, given it is available as per the note above: + +```ruby +with_prometheus_client do |client| + response = client.query('<your query>') + ... +end +``` + +Please refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb) +for how to use its API to query for data. + ## Developing and testing Usage Ping ### 1. Use your Rails console to manually test counters @@ -441,49 +281,591 @@ Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :au ### 2. Generate the SQL query -Your Rails console will give back the generated SQL queries. +Your Rails console will return the generated SQL queries. Example: ```ruby - pry(main)> Gitlab::UsageData.count(User.active) - (0.4ms) SELECT "features"."key" FROM "features" - (0.7ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3)) - (0.6ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3)) - (0.5ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3)) AND "users"."id" BETWEEN 0 AND 99999 +pry(main)> Gitlab::UsageData.count(User.active) + (2.6ms) SELECT "features"."key" FROM "features" + (15.3ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (2.4ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (1.9ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000 ``` ### 3. Optimize queries with #database-lab Paste the SQL query into `#database-lab` to see how the query performs at scale. -- #database-lab is a Slack channel which uses a production-sized environment to test your queries +- `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. - GitLab.com’s production database has a 15 second timeout. -- For each query we require an execution time of under 1 second due do cold caches which can 10x this time. -- Add a specialized index on columns involved to reduce your the execution time. - -In order to have an understanding of the queries execution we add in the MR description the following information - -For counters that have a `time_period` test and add information for both cases. - -- with `time_period = {}` for all time period -- and `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period - -Execution plan and query time before and after optimization - -Using database-lab and [explain.depesz.com](https://explain.depesz.com/) see more details in [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries) +- For each query we require an execution time of under 1 second due to cold caches which can 10x this time. +- Add a specialized index on columns involved to reduce the execution time. -Query generated for the index and time +In order to have an understanding of the query's execution we add in the MR description the following information: -Using database-lab +- For counters that have a `time_period` test we add information for both cases: + - `time_period = {}` for all time periods + - `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period +- Execution plan and query time before and after optimization +- Query generated for the index and time +- Migration output for up and down execution -Migration output for up and down execution +We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries). Examples of query optimization work: - [Example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445) - [Example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871) -### 4. Ask for a Telemetry Review - -On GitLab.com, we have DangerBot setup to monitor Telemetry related files and DangerBot will recommend a Telemetry review. Simply `@gitlab-org/growth/telemetry/engineers` in your MR for a review. +### 4. Add the metric definition + +When adding, changing, or updating metrics, please update the [Usage Statistics definition table](#usage-statistics-definitions). + +### 5. Add new metric to Versions Application + +Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `counts` column. + +### 6. Ask for a Telemetry Review + +On GitLab.com, we have DangerBot setup to monitor Telemetry related files and DangerBot will recommend a Telemetry review. Mention `@gitlab-org/growth/telemetry/engineers` in your MR for a review. + +### Optional: Test Prometheus based Usage Ping + +If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, +then you need to ensure that a Prometheus server is running locally, and that furthermore the respective GitLab components +are exporting metrics to it. If you do not need to test data coming from Prometheus, no further action +is necessary, since Usage Ping should degrade gracefully in the absence of a running Prometheus server. + +There are currently three kinds of components that may export data to Prometheus, and which are included in Usage Ping: + +- [`node_exporter`](https://github.com/prometheus/node_exporter) - Exports node metrics from the host machine +- [`gitlab-exporter`](https://gitlab.com/gitlab-org/gitlab-exporter) - Exports process metrics from various GitLab components +- various GitLab services such as Sidekiq and the Rails server that export their own metrics + +#### Test with an Omnibus container + +This is the recommended approach to test Prometheus based Usage Ping. + +The easiest way to verify your changes is to build a new Omnibus image from your code branch via CI, then download the image +and run a local container instance: + +1. From your merge request, click on the `qa` stage, then trigger the `package-and-qa` job. This job will trigger an Omnibus +build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines). +1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. +1. Open the job logs and locate the full container name including the version. It will take the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`. +1. On your local machine, make sure you are logged in to the GitLab Docker registry. You can find the instructions for this in +[Authenticating to the GitLab Container Registry](../../user/packages/container_registry/index.md#authenticating-to-the-gitlab-container-registry). +1. Once logged in, download the new image via `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>` +1. For more information about working with and running Omnibus GitLab containers in Docker, please refer to [GitLab Docker images](https://docs.gitlab.com/omnibus/docker/README.html) in the Omnibus documentation. + +#### Test with GitLab development toolkits + +This is the less recommended approach, since it comes with a number of difficulties when emulating a real GitLab deployment. + +The [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) is not currently set up to run a Prometheus server or `node_exporter` alongside other GitLab components. If you would +like to do so, [Monitoring the GDK with Prometheus](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/prometheus/index.md#monitoring-the-gdk-with-prometheus) is a good start. + +The [GCK](https://gitlab.com/gitlab-org/gitlab-compose-kit) has limited support for testing Prometheus based Usage Ping. +By default, it already comes with a fully configured Prometheus service that is set up to scrape a number of components, +but with the following limitations: + +- It does not currently run a `gitlab-exporter` instance, so several `process_*` metrics from services such as Gitaly may be missing. +- While it runs a `node_exporter`, `docker-compose` services emulate hosts, meaning that it would normally report itself to not be associated +with any of the other services that are running. That is not how node metrics are reported in a production setup, where `node_exporter` +always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore +appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics will appear in Usage Ping. + +## Usage Statistics definitions + +| Statistic | Section | Stage | Tier | Description | +|:--------------------------------------------------------|:-----------------------------------|:------------|:---------------|:--------------------------------------------------| +| `uuid` | | | | | +| `hostname` | | | | | +| `version` | | | | | +| `installation_type` | | | | | +| `active_user_count` | | | | | +| `recorded_at` | | | | | +| `edition` | | | | | +| `license_md5` | | | | | +| `license_id` | | | | | +| `historical_max_users` | | | | | +| `Name` | `licensee` | | | | +| `Email` | `licensee` | | | | +| `Company` | `licensee` | | | | +| `license_user_count` | | | | | +| `license_starts_at` | | | | | +| `license_expires_at` | | | | | +| `license_plan` | | | | | +| `license_trial` | | | | | +| `assignee_lists` | `counts` | | | | +| `boards` | `counts` | | | | +| `ci_builds` | `counts` | `verify` | | Unique builds in project | +| `ci_internal_pipelines` | `counts` | `verify` | | Total pipelines in GitLab repositories | +| `ci_external_pipelines` | `counts` | `verify` | | Total pipelines in external repositories | +| `ci_pipeline_config_auto_devops` | `counts` | `verify` | | Total pipelines from an Auto DevOps template | +| `ci_pipeline_config_repository` | `counts` | `verify` | | Total Pipelines from templates in repository | +| `ci_runners` | `counts` | `verify` | | Total configured Runners in project | +| `ci_triggers` | `counts` | `verify` | | Total configured Triggers in project | +| `ci_pipeline_schedules` | `counts` | `verify` | | Pipeline schedules in GitLab | +| `auto_devops_enabled` | `counts` |`configure` | | Projects with Auto DevOps template enabled | +| `auto_devops_disabled` | `counts` |`configure` | | Projects with Auto DevOps template disabled | +| `deploy_keys` | `counts` | | | | +| `deployments` | `counts` |`release` | | Total deployments | +| `dast_jobs` | `counts` | | | | +| `successful_deployments` | `counts` |`release` | | Total successful deployments | +| `failed_deployments` | `counts` |`release` | | Total failed deployments | +| `environments` | `counts` |`release` | | Total available and stopped environments | +| `clusters` | `counts` |`configure` | | Total GitLab Managed clusters both enabled and disabled | +| `clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters currently enabled | +| `project_clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters attached to projects| +| `group_clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters attached to groups | +| `instance_clusters_enabled` | `counts` |`configure` | | Total GitLab Managed clusters attached to the instance | +| `clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters | +| `project_clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters previously attached to projects | +| `group_clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters previously attached to groups | +| `instance_clusters_disabled` | `counts` |`configure` | | Total GitLab Managed disabled clusters previously attached to the instance | +| `clusters_platforms_eks` | `counts` |`configure` | | Total GitLab Managed clusters provisioned with GitLab on AWS EKS | +| `clusters_platforms_gke` | `counts` |`configure` | | Total GitLab Managed clusters provisioned with GitLab on GCE GKE | +| `clusters_platforms_user` | `counts` |`configure` | | Total GitLab Managed clusters that are user provisioned | +| `clusters_applications_helm` | `counts` |`configure` | | Total GitLab Managed clusters with Helm enabled | +| `clusters_applications_ingress` | `counts` |`configure` | | Total GitLab Managed clusters with Ingress enabled | +| `clusters_applications_cert_managers` | `counts` |`configure` | | Total GitLab Managed clusters with Cert Manager enabled | +| `clusters_applications_crossplane` | `counts` |`configure` | | Total GitLab Managed clusters with Crossplane enabled | +| `clusters_applications_prometheus` | `counts` |`configure` | | Total GitLab Managed clusters with Prometheus enabled | +| `clusters_applications_runner` | `counts` |`configure` | | Total GitLab Managed clusters with Runner enabled | +| `clusters_applications_knative` | `counts` |`configure` | | Total GitLab Managed clusters with Knative enabled | +| `clusters_applications_elastic_stack` | `counts` |`configure` | | Total GitLab Managed clusters with Elastic Stack enabled | +| `clusters_management_project` | `counts` |`configure` | | Total GitLab Managed clusters with defined cluster management project | +| `in_review_folder` | `counts` | | | | +| `grafana_integrated_projects` | `counts` | | | | +| `groups` | `counts` | | | | +| `issues` | `counts` | | | | +| `issues_created_from_gitlab_error_tracking_ui` | `counts` | `monitor` | | | +| `issues_with_associated_zoom_link` | `counts` | `monitor` | | | +| `issues_using_zoom_quick_actions` | `counts` | `monitor` | | | +| `issues_with_embedded_grafana_charts_approx` | `counts` | `monitor` | | | +| `issues_with_health_status` | `counts` | | | | +| `keys` | `counts` | | | | +| `label_lists` | `counts` | | | | +| `lfs_objects` | `counts` | | | | +| `milestone_lists` | `counts` | | | | +| `milestones` | `counts` | | | | +| `pages_domains` | `counts` |`release` | | Total GitLab Pages domains | +| `pool_repositories` | `counts` | | | | +| `projects` | `counts` | | | | +| `projects_imported_from_github` | `counts` | | | | +| `projects_with_repositories_enabled` | `counts` | | | | +| `projects_with_error_tracking_enabled` | `counts` | `monitor` | | | +| `protected_branches` | `counts` | | | | +| `releases` | `counts` |`release` | | Unique release tags | +| `remote_mirrors` | `counts` | | | | +| `requirements_created` | `counts` | | | | +| `snippets` | `counts` | | | | +| `suggestions` | `counts` | | | | +| `todos` | `counts` | | | | +| `uploads` | `counts` | | | | +| `web_hooks` | `counts` | | | | +| `projects_alerts_active` | `counts` | | | | +| `projects_asana_active` | `counts` | | | | +| `projects_assembla_active` | `counts` | | | | +| `projects_bamboo_active` | `counts` | | | | +| `projects_bugzilla_active` | `counts` | | | | +| `projects_buildkite_active` | `counts` | | | | +| `projects_campfire_active` | `counts` | | | | +| `projects_custom_issue_tracker_active` | `counts` | | | | +| `projects_discord_active` | `counts` | | | | +| `projects_drone_ci_active` | `counts` | | | | +| `projects_emails_on_push_active` | `counts` | | | | +| `projects_external_wiki_active` | `counts` | | | +| `projects_flowdock_active` | `counts` | | | | +| `projects_github_active` | `counts` | | | | +| `projects_hangouts_chat_active` | `counts` | | | | +| `projects_hipchat_active` | `counts` | | | | +| `projects_irker_active` | `counts` | | | | +| `projects_jenkins_active` | `counts` | | | | +| `projects_jira_active` | `counts` | | | | +| `projects_mattermost_active` | `counts` | | | | +| `projects_mattermost_slash_commands_active` | `counts` | | | | +| `projects_microsoft_teams_active` | `counts` | | | | +| `projects_packagist_active` | `counts` | | | | +| `projects_pipelines_email_active` | `counts` | | | | +| `projects_pivotaltracker_active` | `counts` | | | | +| `projects_prometheus_active` | `counts` | | | | +| `projects_pushover_active` | `counts` | | | | +| `projects_redmine_active` | `counts` | | | | +| `projects_slack_active` | `counts` | | | | +| `projects_slack_slash_commands_active` | `counts` | | | | +| `projects_teamcity_active` | `counts` | | | | +| `projects_unify_circuit_active` | `counts` | | | | +| `projects_webex_teams_active` | `counts` | | | | +| `projects_youtrack_active` | `counts` | | | | +| `projects_slack_notifications_active` | `counts` | | | | +| `projects_slack_slash_active` | `counts` | | | | +| `projects_jira_server_active` | `counts` | | | | +| `projects_jira_cloud_active` | `counts` | | | | +| `projects_jira_dvcs_cloud_active` | `counts` | | | | +| `projects_jira_dvcs_server_active` | `counts` | | | | +| `labels` | `counts` | | | | +| `merge_requests` | `counts` | | | | +| `merge_requests_users` | `counts` | | | | +| `notes` | `counts` | | | | +| `wiki_pages_create` | `counts` | | | | +| `wiki_pages_update` | `counts` | | | | +| `wiki_pages_delete` | `counts` | | | | +| `web_ide_commits` | `counts` | | | | +| `web_ide_views` | `counts` | | | | +| `web_ide_merge_requests` | `counts` | | | | +| `web_ide_previews` | `counts` | | | | +| `snippet_comment` | `counts` | | | | +| `commit_comment` | `counts` | | | | +| `merge_request_comment` | `counts` | | | | +| `snippet_create` | `counts` | | | | +| `snippet_update` | `counts` | | | | +| `navbar_searches` | `counts` | | | | +| `cycle_analytics_views` | `counts` | | | | +| `productivity_analytics_views` | `counts` | | | | +| `source_code_pushes` | `counts` | | | | +| `merge_request_create` | `counts` | | | | +| `design_management_designs_create` | `counts` | | | | +| `design_management_designs_update` | `counts` | | | | +| `design_management_designs_delete` | `counts` | | | | +| `licenses_list_views` | `counts` | | | | +| `user_preferences_group_overview_details` | `counts` | | | | +| `user_preferences_group_overview_security_dashboard` | `counts` | | | | +| `ingress_modsecurity_logging` | `counts` | | | | +| `ingress_modsecurity_blocking` | `counts` | | | | +| `ingress_modsecurity_disabled` | `counts` | | | | +| `ingress_modsecurity_not_installed` | `counts` | | | | +| `dependency_list_usages_total` | `counts` | | | | +| `epics` | `counts` | | | | +| `feature_flags` | `counts` | | | | +| `geo_nodes` | `counts` | `geo` | | Number of sites in a Geo deployment | +| `geo_event_log_max_id` | `counts` | `geo` | | Number of replication events on a Geo primary | +| `incident_issues` | `counts` | `monitor` | | Issues created by the alert bot | +| `alert_bot_incident_issues` | `counts` | `monitor` | | Issues created by the alert bot | +| `incident_labeled_issues` | `counts` | `monitor` | | Issues with the incident label | +| `issues_created_gitlab_alerts` | `counts` | `monitor` | | Issues created from alerts by non-alert bot users | +| `issues_created_manually_from_alerts` | `counts` | `monitor` | | Issues created from alerts by non-alert bot users | +| `issues_created_from_alerts` | `counts` | `monitor` | | Issues created from Prometheus and alert management alerts | +| `ldap_group_links` | `counts` | | | | +| `ldap_keys` | `counts` | | | | +| `ldap_users` | `counts` | | | | +| `pod_logs_usages_total` | `counts` | | | | +| `projects_enforcing_code_owner_approval` | `counts` | | | | +| `projects_mirrored_with_pipelines_enabled` | `counts` |`release` | | Projects with repository mirroring enabled | +| `projects_reporting_ci_cd_back_to_github` | `counts` |`verify` | | Projects with a GitHub service pipeline enabled | +| `projects_with_packages` | `counts` |`package` | | Projects with package registry configured | +| `projects_with_prometheus_alerts` | `counts` |`monitor` | | Projects with Prometheus alerting enabled | +| `projects_with_tracing_enabled` | `counts` |`monitor` | | Projects with tracing enabled | +| `projects_with_alerts_service_enabled` | `counts` |`monitor` | | Projects with alerting service enabled | +| `template_repositories` | `counts` | | | | +| `container_scanning_jobs` | `counts` | | | | +| `dependency_scanning_jobs` | `counts` | | | | +| `license_management_jobs` | `counts` | | | | +| `sast_jobs` | `counts` | | | | +| `status_page_projects` | `counts` | `monitor` | | Projects with status page enabled | +| `status_page_issues` | `counts` | `monitor` | | Issues published to a Status Page | +| `status_page_incident_publishes` | `counts` | `monitor` | | Cumulative count of usages of publish operation | +| `status_page_incident_unpublishes` | `counts` | `monitor` | | Cumulative count of usages of unpublish operation | +| `epics_deepest_relationship_level` | `counts` | | | | +| `operations_dashboard_default_dashboard` | `counts` | `monitor` | | Active users with enabled operations dashboard | +| `operations_dashboard_users_with_projects_added` | `counts` | `monitor` | | Active users with projects on operations dashboard| +| `container_registry_enabled` | | | | | +| `dependency_proxy_enabled` | | | | | +| `gitlab_shared_runners_enabled` | | | | | +| `gravatar_enabled` | | | | | +| `ldap_enabled` | | | | | +| `mattermost_enabled` | | | | | +| `omniauth_enabled` | | | | | +| `prometheus_metrics_enabled` | | | | | +| `reply_by_email_enabled` | | | | | +| `average` | `avg_cycle_analytics - code` | | | | +| `sd` | `avg_cycle_analytics - code` | | | | +| `missing` | `avg_cycle_analytics - code` | | | | +| `average` | `avg_cycle_analytics - test` | | | | +| `sd` | `avg_cycle_analytics - test` | | | | +| `missing` | `avg_cycle_analytics - test` | | | | +| `average` | `avg_cycle_analytics - review` | | | | +| `sd` | `avg_cycle_analytics - review` | | | | +| `missing` | `avg_cycle_analytics - review` | | | | +| `average` | `avg_cycle_analytics - staging` | | | | +| `sd` | `avg_cycle_analytics - staging` | | | | +| `missing` | `avg_cycle_analytics - staging` | | | | +| `average` | `avg_cycle_analytics - production` | | | | +| `sd` | `avg_cycle_analytics - production` | | | | +| `missing` | `avg_cycle_analytics - production` | | | | +| `total` | `avg_cycle_analytics` | | | | +| `clusters_applications_cert_managers` | `usage_activity_by_stage` | `configure` | | Unique clusters with certificate managers enabled | +| `clusters_applications_helm` | `usage_activity_by_stage` | `configure` | | Unique clusters with Helm enabled | +| `clusters_applications_ingress` | `usage_activity_by_stage` | `configure` | | Unique clusters with Ingress enabled | +| `clusters_applications_knative` | `usage_activity_by_stage` | `configure` | | Unique clusters with Knative enabled | +| `clusters_management_project` | `usage_activity_by_stage` | `configure` | | Unique clusters with project management enabled | +| `clusters_disabled` | `usage_activity_by_stage` | `configure` | | Total non-"GitLab Managed clusters" | +| `clusters_enabled` | `usage_activity_by_stage` | `configure` | | Total GitLab Managed clusters | +| `clusters_platforms_gke` | `usage_activity_by_stage` | `configure` | | Unique clusters with Google Cloud installed | +| `clusters_platforms_eks` | `usage_activity_by_stage` | `configure` | | Unique clusters with AWS installed | +| `clusters_platforms_user` | `usage_activity_by_stage` | `configure` | | Unique clusters that are user provided | +| `instance_clusters_disabled` | `usage_activity_by_stage` | `configure` | | Unique clusters disabled on instance | +| `instance_clusters_enabled` | `usage_activity_by_stage` | `configure` | | Unique clusters enabled on instance | +| `group_clusters_disabled` | `usage_activity_by_stage` | `configure` | | Unique clusters disabled on group | +| `group_clusters_enabled` | `usage_activity_by_stage` | `configure` | | Unique clusters enabled on group | +| `project_clusters_disabled` | `usage_activity_by_stage` | `configure` | | Unique clusters disabled on project | +| `project_clusters_enabled` | `usage_activity_by_stage` | `configure` | | Unique clusters enabled on project | +| `projects_slack_notifications_active` | `usage_activity_by_stage` | `configure` | | Unique projects with Slack service enabled | +| `projects_slack_slash_active` | `usage_activity_by_stage` | `configure` | | Unique projects with Slack '/' commands enabled | +| `projects_with_prometheus_alerts: 0` | `usage_activity_by_stage` | `monitor` | | Projects with Prometheus enabled and no alerts | +| `deploy_keys` | `usage_activity_by_stage` | `create` | | | +| `keys` | `usage_activity_by_stage` | `create` | | | +| `projects_jira_dvcs_server_active` | `usage_activity_by_stage` | `plan` | | | +| `service_desk_enabled_projects` | `usage_activity_by_stage` | `plan` | | | +| `service_desk_issues` | `usage_activity_by_stage` | `plan` | | | +| `todos: 0` | `usage_activity_by_stage` | `plan` | | | +| `deployments` | `usage_activity_by_stage` | `release` | | Total deployments | +| `failed_deployments` | `usage_activity_by_stage` | `release` | | Total failed deployments | +| `projects_mirrored_with_pipelines_enabled` | `usage_activity_by_stage` | `release` | | Projects with repository mirroring enabled | +| `releases` | `usage_activity_by_stage` | `release` | | Unique release tags in project | +| `successful_deployments: 0` | `usage_activity_by_stage` | `release` | | Total successful deployments | +| `user_preferences_group_overview_security_dashboard: 0` | `usage_activity_by_stage` | `secure` | | | +| `ci_builds` | `usage_activity_by_stage` | `verify` | | Unique builds in project | +| `ci_external_pipelines` | `usage_activity_by_stage` | `verify` | | Total pipelines in external repositories | +| `ci_internal_pipelines` | `usage_activity_by_stage` | `verify` | | Total pipelines in GitLab repositories | +| `ci_pipeline_config_auto_devops` | `usage_activity_by_stage` | `verify` | | Total pipelines from an Auto DevOps template | +| `ci_pipeline_config_repository` | `usage_activity_by_stage` | `verify` | | Pipelines from templates in repository | +| `ci_pipeline_schedules` | `usage_activity_by_stage` | `verify` | | Pipeline schedules in GitLab | +| `ci_pipelines` | `usage_activity_by_stage` | `verify` | | Total pipelines | +| `ci_triggers` | `usage_activity_by_stage` | `verify` | | Triggers enabled | +| `clusters_applications_runner` | `usage_activity_by_stage` | `verify` | | Unique clusters with Runner enabled | +| `projects_reporting_ci_cd_back_to_github: 0` | `usage_activity_by_stage` | `verify` | | Unique projects with a GitHub pipeline enabled | +| `nodes` | `topology` | `enablement`| | The list of server nodes on which GitLab components are running | +| `duration_s` | `topology` | `enablement`| | Time it took to collect topology data | +| `node_memory_total_bytes` | `topology > nodes` | `enablement`| | The total available memory of this node | +| `node_cpus` | `topology > nodes` | `enablement`| | The number of CPU cores of this node | +| `node_services` | `topology > nodes` | `enablement`| | The list of GitLab services running on this node | +| `name` | `topology > nodes > node_services` | `enablement`| | The name of the GitLab service running on this node | +| `process_count` | `topology > nodes > node_services` | `enablement`| | The number of processes running for this service | +| `process_memory_rss` | `topology > nodes > node_services` | `enablement`| | The average Resident Set Size of a service process | +| `process_memory_uss` | `topology > nodes > node_services` | `enablement`| | The average Unique Set Size of a service process | +| `process_memory_pss` | `topology > nodes > node_services` | `enablement`| | The average Proportional Set Size of a service process | + +## Example Usage Ping payload + +The following is example content of the Usage Ping payload. + +```json +{ + "uuid": "0000000-0000-0000-0000-000000000000", + "hostname": "example.com", + "version": "12.10.0-pre", + "installation_type": "omnibus-gitlab", + "active_user_count": 999, + "recorded_at": "2020-04-17T07:43:54.162+00:00", + "edition": "EEU", + "license_md5": "00000000000000000000000000000000", + "license_id": null, + "historical_max_users": 999, + "licensee": { + "Name": "ABC, Inc.", + "Email": "email@example.com", + "Company": "ABC, Inc." + }, + "license_user_count": 999, + "license_starts_at": "2020-01-01", + "license_expires_at": "2021-01-01", + "license_plan": "ultimate", + "license_add_ons": { + }, + "license_trial": false, + "counts": { + "assignee_lists": 999, + "boards": 999, + "ci_builds": 999, + ... + }, + "container_registry_enabled": true, + "dependency_proxy_enabled": false, + "gitlab_shared_runners_enabled": true, + "gravatar_enabled": true, + "influxdb_metrics_enabled": true, + "ldap_enabled": false, + "mattermost_enabled": false, + "omniauth_enabled": true, + "prometheus_metrics_enabled": false, + "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", + "signup_enabled": true, + "web_ide_clientside_preview_enabled": true, + "ingress_modsecurity_enabled": true, + "projects_with_expiration_policy_disabled": 999, + "projects_with_expiration_policy_enabled": 999, + ... + "elasticsearch_enabled": true, + "license_trial_ends_on": null, + "geo_enabled": false, + "git": { + "version": { + "major": 2, + "minor": 26, + "patch": 1 + } + }, + "gitaly": { + "version": "12.10.0-rc1-93-g40980d40", + "servers": 56, + "clusters": 14, + "filesystems": [ + "EXT_2_3_4" + ] + }, + "gitlab_pages": { + "enabled": true, + "version": "1.17.0" + }, + "database": { + "adapter": "postgresql", + "version": "9.6.15" + }, + "app_server": { + "type": "console" + }, + "avg_cycle_analytics": { + "issue": { + "average": 999, + "sd": 999, + "missing": 999 + }, + "plan": { + "average": null, + "sd": 999, + "missing": 999 + }, + "code": { + "average": null, + "sd": 999, + "missing": 999 + }, + "test": { + "average": null, + "sd": 999, + "missing": 999 + }, + "review": { + "average": null, + "sd": 999, + "missing": 999 + }, + "staging": { + "average": null, + "sd": 999, + "missing": 999 + }, + "production": { + "average": null, + "sd": 999, + "missing": 999 + }, + "total": 999 + }, + "usage_activity_by_stage": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "usage_activity_by_stage_monthly": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "topology": { + "nodes": [ + { + "node_memory_total_bytes": 33269903360, + "node_cpus": 16, + "node_services": [ + { + "name": "web", + "process_count": 16, + "process_memory_pss": 233349888, + "process_memory_rss": 788220927, + "process_memory_uss": 195295487 + }, + { + "name": "sidekiq", + "process_count": 1, + "process_memory_pss": 734080000, + "process_memory_rss": 750051328, + "process_memory_uss": 731533312 + }, + ... + ], + ... + }, + ... + ], + "duration_s": 0.013836685999194742 + } +} +``` diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index f0137e542cc..7bb8473117f 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -35,11 +35,18 @@ Here are some things to keep in mind regarding test performance: To run RSpec tests: ```shell -# run all tests +# run test for a file +bin/rspec spec/models/project_spec.rb + +# run test for the example on line 10 on that file +bin/rspec spec/models/project_spec.rb:10 + +# run tests matching the example name has that string +bin/rspec spec/models/project_spec.rb -e associations + +# run all tests, will take hours for GitLab codebase! bin/rspec -# run test for path -bin/rspec spec/[path]/[to]/[spec].rb ``` Use [Guard](https://github.com/guard/guard) to continuously monitor for changes and only run matching tests: @@ -59,7 +66,7 @@ FDOC=1 bin/rspec spec/[path]/[to]/[spec].rb ### General guidelines -- Use a single, top-level `describe ClassName` block. +- Use a single, top-level `RSpec.describe ClassName` block. - Use `.method` to describe class methods and `#method` to describe instance methods. - Use `context` to test branching logic. @@ -323,7 +330,7 @@ Feature.enabled?(:ci_live_trace) # => false If you wish to set up a test where a feature flag is enabled only for some actors and not others, you can specify this in options passed to the helper. For example, to enable the `ci_live_trace` -feature flag for a specifc project: +feature flag for a specific project: ```ruby project1, project2 = build_list(:project, 2) @@ -340,7 +347,7 @@ This represents an actual behavior of FlipperGate: 1. You can enable an override for a specified actor to be enabled 1. You can disable (remove) an override for a specified actor, - fallbacking to default state + falling back to default state 1. There's no way to model that you explicitly disable a specified actor ```ruby @@ -357,6 +364,60 @@ Feature.enabled?(:my_feature2) # => false Feature.enabled?(:my_feature2, project1) # => true ``` +#### `stub_feature_flags` vs `Feature.enable*` + +It is preferred to use `stub_feature_flags` for enabling feature flags +in testing environment. This method provides a simple and well described +interface for a simple use-cases. + +However, in some cases a more complex behaviors needs to be tested, +like a feature flag percentage rollouts. This can be achieved using +the `.enable_percentage_of_time` and `.enable_percentage_of_actors` + +```ruby +# Good: feature needs to be explicitly disabled, as it is enabled by default if not defined +stub_feature_flags(my_feature: false) +stub_feature_flags(my_feature: true) +stub_feature_flags(my_feature: project) +stub_feature_flags(my_feature: [project, project2]) + +# Bad +Feature.enable(:my_feature_2) + +# Good: enable my_feature for 50% of time +Feature.enable_percentage_of_time(:my_feature_3, 50) + +# Good: enable my_feature for 50% of actors/gates/things +Feature.enable_percentage_of_actors(:my_feature_4, 50) +``` + +Each feature flag that has a defined state will be persisted +for test execution time: + +```ruby +Feature.persisted_names.include?('my_feature') => true +Feature.persisted_names.include?('my_feature_2') => true +Feature.persisted_names.include?('my_feature_3') => true +Feature.persisted_names.include?('my_feature_4') => true +``` + +#### Stubbing gate + +It is required that a gate that is passed as an argument to `Feature.enabled?` +and `Feature.disabled?` is an object that includes `FeatureGate`. + +In specs you can use a `stub_feature_flag_gate` method that allows you to have +quickly your custom gate: + +```ruby +gate = stub_feature_flag_gate('CustomActor') + +stub_feature_flags(ci_live_trace: gate) + +Feature.enabled?(:ci_live_trace) # => false +Feature.enabled?(:ci_live_trace, gate) # => true +``` + ### Pristine test environments The code exercised by a single GitLab test may access and modify many items of @@ -406,7 +467,7 @@ However, if a spec makes direct Redis calls, it should mark itself with the #### Background jobs / Sidekiq By default, Sidekiq jobs are enqueued into a jobs array and aren't processed. -If a test enqueues Sidekiq jobs and need them to be processed, the +If a test queues Sidekiq jobs and need them to be processed, the `:sidekiq_inline` trait can be used. The `:sidekiq_might_not_need_inline` trait was added when [Sidekiq inline mode was @@ -662,7 +723,7 @@ module Spec end ``` -Helpers should not change the RSpec config. For instance, the helpers module +Helpers should not change the RSpec configuration. For instance, the helpers module described above should not include: ```ruby @@ -723,9 +784,9 @@ end This will create a repository containing two files, with default permissions and the specified content. -### Config +### Configuration -RSpec config files are files that change the RSpec config (i.e. +RSpec configuration files are files that change the RSpec configuration (i.e. `RSpec.configure do |config|` blocks). They should be placed under `spec/support/`. @@ -744,7 +805,7 @@ RSpec.configure do |config| end ``` -If a config file only consists of `config.include`, you can add these +If a configuration file only consists of `config.include`, you can add these `config.include` directly in `spec/spec_helper.rb`. For very generic helpers, consider including them in the `spec/support/rspec.rb` diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 57cfcf34726..7df3cd614c7 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -91,7 +91,7 @@ point of failure and so the screenshot would not be captured at the right moment All tests expect to be able to log in at the start of the test. -For an example see: <https://gitlab.com/gitlab-org/gitlab/issues/34736> +For an example see: <https://gitlab.com/gitlab-org/gitlab/-/issues/34736> Ideally, any actions performed in an `after(:context)` (or [`before(:context)`](#limit-the-use-of-the-ui-in-beforecontext-and-after-hooks)) block would be performed via the API. But if it's necessary to do so via the UI (e.g., if API functionality doesn't exist), make sure to log out at the end of the block. diff --git a/doc/development/testing_guide/end_to_end/feature_flags.md b/doc/development/testing_guide/end_to_end/feature_flags.md index 3bd07f17207..ada20cc9dad 100644 --- a/doc/development/testing_guide/end_to_end/feature_flags.md +++ b/doc/development/testing_guide/end_to_end/feature_flags.md @@ -26,4 +26,4 @@ end It's also possible to run an entire scenario with a feature flag enabled, without having to edit existing tests or write new ones. -Please see the [QA readme](https://gitlab.com/gitlab-org/gitlab/tree/master/qa#running-tests-with-a-feature-flag-enabled) for details. +Please see the [QA README](https://gitlab.com/gitlab-org/gitlab/tree/master/qa#running-tests-with-a-feature-flag-enabled) for details. diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index e6a683e9148..ac051b827d2 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -65,18 +65,18 @@ subgraph "gitlab-qa-mirror pipeline" end ``` -1. Developer triggers a manual action, that can be found in CE / EE merge +1. Developer triggers a manual action, that can be found in GitLab merge requests. This starts a chain of pipelines in multiple projects. 1. The script being executed triggers a pipeline in - [Omnibus GitLab Mirror](https://gitlab.com/gitlab-org/omnibus-gitlab-mirror) + [Omnibus GitLab Mirror](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror) and waits for the resulting status. We call this a _status attribution_. -1. GitLab packages are being built in the [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab) +1. GitLab packages are being built in the [Omnibus GitLab Mirror](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror) pipeline. Packages are then pushed to its Container Registry. 1. When packages are ready, and available in the registry, a final step in the - [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab) pipeline, triggers a new + [Omnibus GitLab Mirror](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror) pipeline, triggers a new GitLab QA pipeline (those with access can view them at `https://gitlab.com/gitlab-org/gitlab-qa-mirror/pipelines`). It also waits for a resulting status. 1. GitLab QA pulls images from the registry, spins-up containers and runs tests @@ -84,9 +84,9 @@ subgraph "gitlab-qa-mirror pipeline" tool. 1. The result of the GitLab QA pipeline is being - propagated upstream, through Omnibus, back to the CE / EE merge request. + propagated upstream, through Omnibus, back to the GitLab merge request. -Please note, we plan to [add more specific information](https://gitlab.com/gitlab-org/quality/team-tasks/issues/156) +Please note, we plan to [add more specific information](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/156) about the tests included in each job/scenario that runs in `gitlab-qa-mirror`. #### With Pipeline for Merged Results @@ -132,7 +132,7 @@ as well as these: | `QA_RSPEC_TAGS` | The RSpec tags to add (no default) | For now [manual jobs with custom variables will not use the same variable -when retried](https://gitlab.com/gitlab-org/gitlab/issues/31367), so if you want to run the same test(s) multiple times, +when retried](https://gitlab.com/gitlab-org/gitlab/-/issues/31367), so if you want to run the same test(s) multiple times, specify the same variables in each `custom-parallel` job (up to as many of the 10 available jobs that you want to run). @@ -155,7 +155,7 @@ See [Review Apps](../review_apps.md) for more details about Review Apps. If you are not [testing code in a merge request](#testing-code-in-merge-requests), there are two main options for running the tests. If you simply want to run -the existing tests against a live GitLab instance or against a pre-built docker image +the existing tests against a live GitLab instance or against a pre-built Docker image you can use the [GitLab QA orchestrator](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md). See also [examples of the test scenarios you can run via the orchestrator](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/what_tests_can_be_run.md#examples). @@ -191,5 +191,5 @@ Continued reading: You can ask question in the `#quality` channel on Slack (GitLab internal) or you can find an issue you would like to work on in -[the `gitlab` issue tracker](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=QA&label_name%5B%5D=test), or -[the `gitlab-qa` issue tracker](https://gitlab.com/gitlab-org/gitlab-qa/issues?label_name%5B%5D=new+scenario). +[the `gitlab` issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues?label_name%5B%5D=QA&label_name%5B%5D=test), or +[the `gitlab-qa` issue tracker](https://gitlab.com/gitlab-org/gitlab-qa/-/issues?label_name%5B%5D=new+scenario). diff --git a/doc/development/testing_guide/end_to_end/page_objects.md b/doc/development/testing_guide/end_to_end/page_objects.md index 9d4fa5316c4..d43d88779c7 100644 --- a/doc/development/testing_guide/end_to_end/page_objects.md +++ b/doc/development/testing_guide/end_to_end/page_objects.md @@ -89,7 +89,7 @@ end ### Defining Elements -The `view` DSL method will correspond to the rails View, partial, or vue component that renders the elements. +The `view` DSL method will correspond to the rails View, partial, or Vue component that renders the elements. The `element` DSL method in turn declares an element for which a corresponding `data-qa-selector=element_name_snaked` data attribute will need to be added to the view file. @@ -134,7 +134,7 @@ view 'app/views/my/view.html.haml' do end ``` -To add these elements to the view, you must change the rails View, partial, or vue component by adding a `data-qa-selector` attribute +To add these elements to the view, you must change the rails View, partial, or Vue component by adding a `data-qa-selector` attribute for each element defined. In our case, `data-qa-selector="login_field"`, `data-qa-selector="password_field"` and `data-qa-selector="sign_in_button"` @@ -149,7 +149,7 @@ In our case, `data-qa-selector="login_field"`, `data-qa-selector="password_field Things to note: -- The name of the element and the qa_selector must match and be snake_cased +- The name of the element and the `qa_selector` must match and be snake_cased - If the element appears on the page unconditionally, add `required: true` to the element. See [Dynamic element validation](dynamic_element_validation.md) - You may see `.qa-selector` classes in existing Page Objects. We should prefer the [`data-qa-selector`](#data-qa-selector-vs-qa-selector) @@ -255,7 +255,7 @@ These steps ensure the sanity selectors check will detect problems properly. For example, `qa/qa/ee/page/merge_request/show.rb` adds EE-specific methods to `qa/qa/page/merge_request/show.rb` (with `QA::Page::MergeRequest::Show.prepend_if_ee('QA::EE::Page::MergeRequest::Show')`) and following is how it's implemented -(only showing the relevant part and refering to the 4 steps described above with inline comments): +(only showing the relevant part and referring to the 4 steps described above with inline comments): ```ruby module QA diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index b7c93d205a3..b1a8a14163c 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -9,10 +9,11 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec |-----|-------------| | `:elasticsearch` | The test requires an Elasticsearch service. It is used by the [instance-level scenario](https://gitlab.com/gitlab-org/gitlab-qa#definitions) [`Test::Integration::Elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/72b62b51bdf513e2936301cb6c7c91ec27c35b4d/qa/qa/ee/scenario/test/integration/elasticsearch.rb) to include only tests that require Elasticsearch. | | `:kubernetes` | The test includes a GitLab instance that is configured to be run behind an SSH tunnel, allowing a TLS-accessible GitLab. This test will also include provisioning of at least one Kubernetes cluster to test against. *This tag is often be paired with `:orchestrated`.* | -| `:orchestrated` | The GitLab instance under test may be [configured by `gitlab-qa`](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#orchestrated-tests) to be different to the default GitLab configuration, or `gitlab-qa` may launch additional services in separate docker containers, or both. Tests tagged with `:orchestrated` are excluded when testing environments where we can't dynamically modify GitLab's configuration (for example, Staging). | +| `:orchestrated` | The GitLab instance under test may be [configured by `gitlab-qa`](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#orchestrated-tests) to be different to the default GitLab configuration, or `gitlab-qa` may launch additional services in separate Docker containers, or both. Tests tagged with `:orchestrated` are excluded when testing environments where we can't dynamically modify GitLab's configuration (for example, Staging). | | `:quarantine` | The test has been [quarantined](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests), will run in a separate job that only includes quarantined tests, and is allowed to fail. The test will be skipped in its regular job so that if it fails it will not hold up the pipeline. | | `:reliable` | The test has been [promoted to a reliable test](https://about.gitlab.com/handbook/engineering/quality/guidelines/reliable-tests/#promoting-an-existing-test-to-reliable) meaning it passes consistently in all pipelines, including merge requests. | | `:requires_admin` | The test requires an admin account. Tests with the tag are excluded when run against Canary and Production environments. | | `:runner` | The test depends on and will set up a GitLab Runner instance, typically to run a pipeline. | | `:gitaly_ha` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | | `:skip_live_env` | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. | +| `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. diff --git a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md index f360226d922..77d820e1686 100644 --- a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md +++ b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md @@ -2,13 +2,13 @@ ## Jenkins spec -The [`jenkins_build_status_spec`](https://gitlab.com/gitlab-org/gitlab/blob/163c8a8c814db26d11e104d1cb2dcf02eb567dbe/qa/qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb) spins up a Jenkins instance in a docker container based on an image stored in the [GitLab-QA container registry](https://gitlab.com/gitlab-org/gitlab-qa/container_registry). -The docker image it uses is preconfigured with some base data and plugins. +The [`jenkins_build_status_spec`](https://gitlab.com/gitlab-org/gitlab/blob/163c8a8c814db26d11e104d1cb2dcf02eb567dbe/qa/qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb) spins up a Jenkins instance in a Docker container based on an image stored in the [GitLab-QA container registry](https://gitlab.com/gitlab-org/gitlab-qa/container_registry). +The Docker image it uses is preconfigured with some base data and plugins. The test then configures the GitLab plugin in Jenkins with a URL of the GitLab instance that will be used to run the tests. Unfortunately, the GitLab Jenkins plugin does not accept ports so `http://localhost:3000` would -not be accepted. Therefore, this requires us to run GitLab on port 80 or inside a docker container. +not be accepted. Therefore, this requires us to run GitLab on port 80 or inside a Docker container. -To start a docker container for GitLab based on the nightly image: +To start a Docker container for GitLab based on the nightly image: ```shell docker run \ @@ -24,7 +24,7 @@ To run the tests from the `/qa` directory: CHROME_HEADLESS=false bin/qa Test::Instance::All http://localhost -- qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb ``` -The test will automatically spinup a docker container for Jenkins and tear down once the test completes. +The test will automatically spin up a Docker container for Jenkins and tear down once the test completes. However, if you need to run Jenkins manually outside of the tests, use this command: @@ -46,5 +46,5 @@ only to prevent it from running in the pipelines for live environments such as S ### Troubleshooting -If Jenkins docker container exits without providing any information in the logs, try increasing the memory used by +If Jenkins Docker container exits without providing any information in the logs, try increasing the memory used by the Docker Engine. diff --git a/doc/development/testing_guide/end_to_end/style_guide.md b/doc/development/testing_guide/end_to_end/style_guide.md index 9c02af12d5d..7e9f097f624 100644 --- a/doc/development/testing_guide/end_to_end/style_guide.md +++ b/doc/development/testing_guide/end_to_end/style_guide.md @@ -49,7 +49,7 @@ Notice that in the above example, before clicking the `:operations_environments_ When adding new elements to a page, it's important that we have a uniform element naming convention. -We follow a simple formula roughly based on hungarian notation. +We follow a simple formula roughly based on Hungarian notation. *Formula*: `element :<descriptor>_<type>` @@ -109,7 +109,7 @@ we use the name of the page object in [snake_case](https://en.wikipedia.org/wiki (all lowercase, with words separated by an underscore). See good and bad examples below. While we prefer to follow the standard in most cases, it is also acceptable to -use common abbreviations (e.g., mr) or other alternatives, as long as +use common abbreviations (e.g., `mr`) or other alternatives, as long as the name is not ambiguous. This can include appending `_page` if it helps to avoid confusion or make the code more readable. For example, if a page object is named `New`, it could be confusing to name the block argument `new` because that @@ -123,7 +123,7 @@ Capybara DSL, potentially leading to confusion and bugs. **Good** ```ruby -Page::Project::Settings::Members.perform do |members| +Page::Project::Members.perform do |members| members.do_something end ``` @@ -149,7 +149,7 @@ end **Bad** ```ruby -Page::Project::Settings::Members.perform do |project_settings_members_page| +Page::Project::Members.perform do |project_settings_members_page| project_settings_members_page.do_something end ``` diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md index 6c1b06ce59a..7aed908c4f6 100644 --- a/doc/development/testing_guide/flaky_tests.md +++ b/doc/development/testing_guide/flaky_tests.md @@ -49,20 +49,20 @@ examples in a JSON report file on `master` (`retrieve-tests-metadata` and This was originally implemented in: <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/13021>. -If you want to enable retries locally, you can use the `RETRIES` env variable. +If you want to enable retries locally, you can use the `RETRIES` environment variable. For instance `RETRIES=1 bin/rspec ...` would retry the failing examples once. ## Problems we had in the past at GitLab -- [`rspec-retry` is biting us when some API specs fail](https://gitlab.com/gitlab-org/gitlab-foss/issues/29242): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9825> -- [Sporadic RSpec failures due to `PG::UniqueViolation`](https://gitlab.com/gitlab-org/gitlab-foss/issues/28307#note_24958837): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9846> +- [`rspec-retry` is biting us when some API specs fail](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29242): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9825> +- [Sporadic RSpec failures due to `PG::UniqueViolation`](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28307#note_24958837): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9846> - Follow-up: <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10688> - - [Capybara.reset_session! should be called before requests are blocked](https://gitlab.com/gitlab-org/gitlab-foss/issues/33779): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12224> + - [Capybara.reset_session! should be called before requests are blocked](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/33779): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12224> - FFaker generates funky data that tests are not ready to handle (and tests should be predictable so that's bad!): - - [Make `spec/mailers/notify_spec.rb` more robust](https://gitlab.com/gitlab-org/gitlab-foss/issues/20121): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10015> - - [Transient failure in `spec/requests/api/commits_spec.rb`](https://gitlab.com/gitlab-org/gitlab-foss/issues/27988#note_25342521): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9944> - - [Replace FFaker factory data with sequences](https://gitlab.com/gitlab-org/gitlab-foss/issues/29643): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10184> - - [Transient failure in spec/finders/issues_finder_spec.rb](https://gitlab.com/gitlab-org/gitlab-foss/issues/30211#note_26707685): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10404> + - [Make `spec/mailers/notify_spec.rb` more robust](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/20121): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10015> + - [Transient failure in `spec/requests/api/commits_spec.rb`](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/27988#note_25342521): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9944> + - [Replace FFaker factory data with sequences](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29643): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10184> + - [Transient failure in spec/finders/issues_finder_spec.rb](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30211#note_26707685): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10404> ### Time-sensitive flaky tests @@ -75,24 +75,24 @@ For instance `RETRIES=1 bin/rspec ...` would retry the failing examples once. ### Feature tests -- [Be sure to create all the data the test need before starting exercise](https://gitlab.com/gitlab-org/gitlab-foss/issues/32622#note_31128195): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12059> -- [Bis](https://gitlab.com/gitlab-org/gitlab-foss/issues/34609#note_34048715): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12604> -- [Bis](https://gitlab.com/gitlab-org/gitlab-foss/issues/34698#note_34276286): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12664> -- [Assert against the underlying database state instead of against a page's content](https://gitlab.com/gitlab-org/gitlab-foss/issues/31437): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10934> -- In JS tests, shifting elements can cause Capybara to misclick when the element moves at the exact time Capybara sends the click +- [Be sure to create all the data the test need before starting exercise](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/32622#note_31128195): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12059> +- [Bis](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34609#note_34048715): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12604> +- [Bis](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34698#note_34276286): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12664> +- [Assert against the underlying database state instead of against a page's content](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/31437): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10934> +- In JS tests, shifting elements can cause Capybara to mis-click when the element moves at the exact time Capybara sends the click - [Dropdowns rendering upward or downward due to window size and scroll position](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17660) - - [Lazy loaded images can cause Capybara to misclick](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18713) + - [Lazy loaded images can cause Capybara to mis-click](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18713) - [Triggering JS events before the event handlers are set up](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18742) -- [Wait for the image to be lazy-loaded when asserting on a Markdown image's src attribute](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25408) +- [Wait for the image to be lazy-loaded when asserting on a Markdown image's `src` attribute](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25408) #### Capybara viewport size related issues -- [Transient failure of spec/features/issues/filtered_search/filter_issues_spec.rb](https://gitlab.com/gitlab-org/gitlab-foss/issues/29241#note_26743936): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10411> +- [Transient failure of spec/features/issues/filtered_search/filter_issues_spec.rb](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29241#note_26743936): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10411> #### Capybara JS driver related issues -- [Don't wait for AJAX when no AJAX request is fired](https://gitlab.com/gitlab-org/gitlab-foss/issues/30461): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10454> -- [Bis](https://gitlab.com/gitlab-org/gitlab-foss/issues/34647): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12626> +- [Don't wait for AJAX when no AJAX request is fired](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30461): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10454> +- [Bis](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34647): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12626> #### PhantomJS / WebKit related issues diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 52d538c7159..37e1066e7aa 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -26,7 +26,7 @@ Jest tests can be found in `/spec/frontend` and `/ee/spec/frontend` in EE. > **Note:** > -> Most examples have a Jest and Karma example. See the Karma examples only as explanation to what's going on in the code, should you stumble over some usescases during your discovery. The Jest examples are the one you should follow. +> Most examples have a Jest and Karma example. See the Karma examples only as explanation to what's going on in the code, should you stumble over some use cases during your discovery. The Jest examples are the one you should follow. ## Karma test suite @@ -61,7 +61,7 @@ which could arise (especially with testing against browser specific features). - Jest runs in a Node.js environment, not in a browser. Support for running Jest tests in a browser [is planned](https://gitlab.com/gitlab-org/gitlab/-/issues/26982). - Because Jest runs in a Node.js environment, it uses [jsdom](https://github.com/jsdom/jsdom) by default. See also its [limitations](#limitations-of-jsdom) below. - Jest does not have access to Webpack loaders or aliases. - The aliases used by Jest are defined in its [own config](https://gitlab.com/gitlab-org/gitlab/blob/master/jest.config.js). + The aliases used by Jest are defined in its [own configuration](https://gitlab.com/gitlab-org/gitlab/blob/master/jest.config.js). - All calls to `setTimeout` and `setInterval` are mocked away. See also [Jest Timer Mocks](https://jestjs.io/docs/en/timer-mocks). - `rewire` is not required because Jest supports mocking modules. See also [Manual Mocks](https://jestjs.io/docs/en/manual-mocks). - No [context object](https://jasmine.github.io/tutorials/your_first_suite#section-The_%3Ccode%3Ethis%3C/code%3E_keyword) is passed to tests in Jest. @@ -200,15 +200,15 @@ For example, it's better to use the generated markup to trigger a button click a ## Common practices -Following you'll find some general common practices you will find as part of our testsuite. Should you stumble over something not following this guide, ideally fix it right away. 🎉 +Following you'll find some general common practices you will find as part of our test suite. Should you stumble over something not following this guide, ideally fix it right away. 🎉 ### How to query DOM elements -When it comes to querying DOM elements in your tests, it is best to uniquely target the element, without adding additional attributes specifically for testing purposes. Sometimes this cannot be done feasibly. In these cases, adding test attributes to simplify the selectors might be the best option. +When it comes to querying DOM elements in your tests, it is best to uniquely and semantically target the element. Sometimes this cannot be done feasibly. In these cases, adding test attributes to simplify the selectors might be the best option. Preferentially, in component testing with `@vue/test-utils`, you should query for child components using the component itself. This helps enforce that specific behavior can be covered by that component's individual unit tests. Otherwise, try to use: -- A behavioral attribute like `name` (also verifies that `name` was setup properly) +- A semantic attribute like `name` (also verifies that `name` was setup properly) - A `data-testid` attribute ([recommended by maintainers of `@vue/test-utils`](https://github.com/vuejs/vue-test-utils/issues/1498#issuecomment-610133465)) - a Vue `ref` (if using `@vue/test-utils`) @@ -216,11 +216,17 @@ Examples: ```javascript it('exists', () => { + // Good wrapper.find(FooComponent); wrapper.find('input[name=foo]'); wrapper.find('[data-testid="foo"]'); wrapper.find({ ref: 'foo'}); + + // Bad wrapper.find('.js-foo'); + wrapper.find('.btn-primary'); + wrapper.find('.qa-foo-component'); + wrapper.find('[data-qa-selector="foo"]'); }); ``` @@ -556,7 +562,7 @@ The more challenging part are mocks, which can be used for functions or even dep ### Manual module mocks Manual mocks are used to mock modules across the entire Jest environment. This is a very powerful testing tool that helps simplify -unit testing by mocking out modules which cannot be easily consumned in our test environment. +unit testing by mocking out modules which cannot be easily consumed in our test environment. > **WARNING:** Do not use manual mocks if a mock should not be consistently applied in every spec (i.e. it's only needed by a few specs). > Instead, consider using [`jest.mock(..)`](https://jestjs.io/docs/en/jest-object#jestmockmodulename-factory-options) @@ -582,10 +588,10 @@ If a manual mock is needed for a CE module, please place it in `spec/frontend/mo - [`mocks/axios_utils`](https://gitlab.com/gitlab-org/gitlab/blob/bd20aeb64c4eed117831556c54b40ff4aee9bfd1/spec/frontend/mocks/ce/lib/utils/axios_utils.js#L1) - This mock is helpful because we don't want any unmocked requests to pass any tests. Also, we are able to inject some test helpers such as `axios.waitForAll`. - [`__mocks__/mousetrap/index.js`](https://gitlab.com/gitlab-org/gitlab/blob/cd4c086d894226445be9d18294a060ba46572435/spec/frontend/__mocks__/mousetrap/index.js#L1) - - This mock is helpful because the module itself uses amd format which webpack understands, but is incompatible with the jest environment. This mock doesn't remove + This mock is helpful because the module itself uses AMD format which webpack understands, but is incompatible with the jest environment. This mock doesn't remove any behavior, only provides a nice es6 compatible wrapper. - [`__mocks__/monaco-editor/index.js`](https://gitlab.com/gitlab-org/gitlab/blob/b7f914cddec9fc5971238cdf12766e79fa1629d7/spec/frontend/__mocks__/monaco-editor/index.js) - - This mock is helpful because the monaco package is completely incompatible in a Jest environment. In fact, webpack requires a special loader to make it work. This mock + This mock is helpful because the Monaco package is completely incompatible in a Jest environment. In fact, webpack requires a special loader to make it work. This mock simply makes this package consumable by Jest. ### Keep mocks light @@ -611,7 +617,7 @@ As long as the fixtures don't change, `yarn test` is sufficient (and saves you s ### Live testing and focused testing -- Jest -While you work on a testsuite, you may want to run these specs in watch mode, so they rerun automatically on every save. +While you work on a test suite, you may want to run these specs in watch mode, so they rerun automatically on every save. ```shell # Watch and rerun all specs matching the name icon @@ -801,9 +807,9 @@ Tests relevant for frontend development can be found at the following places: RSpec runs complete [feature tests](testing_levels.md#frontend-feature-tests), while the Jest and Karma directories contain [frontend unit tests](testing_levels.md#frontend-unit-tests), [frontend component tests](testing_levels.md#frontend-component-tests), and [frontend integration tests](testing_levels.md#frontend-integration-tests). -All tests in `spec/javascripts/` will eventually be migrated to `spec/frontend/` (see also [#52483](https://gitlab.com/gitlab-org/gitlab-foss/issues/52483)). +All tests in `spec/javascripts/` will eventually be migrated to `spec/frontend/` (see also [#52483](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52483)). -Before May 2018, `features/` also contained feature tests run by Spinach. These tests were removed from the codebase in May 2018 ([#23036](https://gitlab.com/gitlab-org/gitlab-foss/issues/23036)). +Before May 2018, `features/` also contained feature tests run by Spinach. These tests were removed from the codebase in May 2018 ([#23036](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/23036)). See also [Notes on testing Vue components](../fe_guide/vue.md#testing-vue-components). @@ -830,11 +836,11 @@ testAction( ); ``` -Check an example in [spec/javascripts/ide/stores/actions_spec.jsspec/javascripts/ide/stores/actions_spec.js](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/javascripts/ide/stores/actions_spec.js). +Check an example in [`spec/javascripts/ide/stores/actions_spec.jsspec/javascripts/ide/stores/actions_spec.js`](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/javascripts/ide/stores/actions_spec.js). -### Wait until axios requests finish +### Wait until Axios requests finish -The axios utils mock module located in `spec/frontend/mocks/ce/lib/utils/axios_utils.js` contains two helper methods for Jest tests that spawn HTTP requests. +The Axios Utils mock module located in `spec/frontend/mocks/ce/lib/utils/axios_utils.js` contains two helper methods for Jest tests that spawn HTTP requests. These are very useful if you don't have a handle to the request's Promise, for example when a Vue component does a request as part of its life cycle. - `waitFor(url, callback)`: Runs `callback` after a request to `url` finishes (either successfully or unsuccessfully). @@ -844,11 +850,11 @@ Both functions run `callback` on the next tick after the requests finish (using ## Testing with older browsers -Some regressions only affect a specific browser version. We can install and test in particular browsers with either Firefox or Browserstack using the following steps: +Some regressions only affect a specific browser version. We can install and test in particular browsers with either Firefox or BrowserStack using the following steps: -### Browserstack +### BrowserStack -[Browserstack](https://www.browserstack.com/) allows you to test more than 1200 mobile devices and browsers. +[BrowserStack](https://www.browserstack.com/) allows you to test more than 1200 mobile devices and browsers. You can use it directly through the [live app](https://www.browserstack.com/live) or you can install the [chrome extension](https://chrome.google.com/webstore/detail/browserstack/nkihdmlheodkdfojglpcjjmioefjahjb) for easy access. You can find the credentials on 1Password, under `frontendteam@gitlab.com`. @@ -860,7 +866,7 @@ You can download any older version of Firefox from the releases FTP server, <htt 1. From the website, select a version, in this case `50.0.1`. 1. Go to the mac folder. -1. Select your preferred language, you will find the dmg package inside, download it. +1. Select your preferred language, you will find the DMG package inside, download it. 1. Drag and drop the application to any other folder but the `Applications` folder. 1. Rename the application to something like `Firefox_Old`. 1. Move the application to the `Applications` folder. diff --git a/doc/development/testing_guide/index.md b/doc/development/testing_guide/index.md index f0fb06910f8..0d470e0e737 100644 --- a/doc/development/testing_guide/index.md +++ b/doc/development/testing_guide/index.md @@ -20,7 +20,7 @@ Following are two great articles that everyone should read to understand what automated testing means, and what are its principles: - [Five Factor Testing](https://madeintandem.com/blog/five-factor-testing/): Why do we need tests? -- [Principles of Automated Testing](http://www.lihaoyi.com/post/PrinciplesofAutomatedTesting.html): Levels of testing. Prioritize tests. Cost of tests. +- [Principles of Automated Testing](https://www.lihaoyi.com/post/PrinciplesofAutomatedTesting.html): Levels of testing. Prioritize tests. Cost of tests. ## [Testing levels](testing_levels.md) @@ -58,7 +58,7 @@ Everything you should know about how to test Rake tasks. ## [End-to-end tests](end_to_end/index.md) Everything you should know about how to run end-to-end tests using -[GitLab QA](ttps://gitlab.com/gitlab-org/gitlab-qa) testing framework. +[GitLab QA](https://gitlab.com/gitlab-org/gitlab-qa) testing framework. ## [Migrations tests](testing_migrations_guide.md) diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 58acf937d77..3d7aea89e73 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -55,8 +55,8 @@ subgraph "CNG-mirror pipeline" each component (e.g. `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the commit from the [GitLab pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) and stores them in its [registry](https://gitlab.com/gitlab-org/build/CNG-mirror/container_registry). - - We use the [`CNG-mirror`](https://gitlab.com/gitlab-org/build/CNG-mirror) project so that the `CNG`, (**C**loud - **N**ative **G**itLab), project's registry is not overloaded with a + - We use the [`CNG-mirror`](https://gitlab.com/gitlab-org/build/CNG-mirror) project so that the `CNG`, (Cloud + Native GitLab), project's registry is not overloaded with a lot of transient Docker images. - Note that the official CNG images are built by the `cloud-native-image` job, which runs only for tags, and triggers itself a [`CNG`](https://gitlab.com/gitlab-org/build/CNG) pipeline. @@ -139,8 +139,8 @@ browser performance testing using a The `review-apps-ee` and `review-apps-ce` clusters are currently set up with the following node pools: -- `review-apps-ee` of preemptible `e2-highcpu-16` (16 vCPU, 16 GB memory) nodes with autoscaling -- `review-apps-ce` of preemptible `n1-standard-8` (8 vCPU, 16 GB memory) nodes with autoscaling +- `review-apps-ee` of pre-emptible `e2-highcpu-16` (16 vCPU, 16 GB memory) nodes with autoscaling +- `review-apps-ce` of pre-emptible `n1-standard-8` (8 vCPU, 16 GB memory) nodes with autoscaling ### Helm @@ -152,7 +152,7 @@ used by the `review-deploy` and `review-stop` jobs. ### Get access to the GCP Review Apps cluster -You need to [open an access request (internal link)](https://gitlab.com/gitlab-com/access-requests/issues/new) +You need to [open an access request (internal link)](https://gitlab.com/gitlab-com/access-requests/-/issues/new) for the `gcp-review-apps-sg` GCP group. In order to join a group, you must specify the desired GCP role in your access request. The role is what will grant you specific permissions in order to engage with Review App containers. @@ -212,7 +212,7 @@ If [Review App Stability](https://app.periscopedata.com/app/gitlab/496118/Engine dips this may be a signal that the `review-apps-ce/ee` cluster is unhealthy. Leading indicators may be health check failures leading to restarts or majority failure for Review App deployments. -The [Review Apps Overview dashboard](https://app.google.stackdriver.com/dashboards/6798952013815386466?project=gitlab-review-apps&timeDomain=1d) +The [Review Apps Overview dashboard](https://console.cloud.google.com/monitoring/classic/dashboards/6798952013815386466?project=gitlab-review-apps&timeDomain=1d) aids in identifying load spikes on the cluster, and if nodes are problematic or the entire cluster is trending towards unhealthy. ### Release failed with `ImagePullBackOff` @@ -278,14 +278,14 @@ kubectl top pods | sort --key 2 --numeric **Potential cause:** -This could be a sign that there are too many stale secrets and/or config maps. +This could be a sign that there are too many stale secrets and/or configuration maps. **Where to look for further debugging:** Look at [the list of Configurations](https://console.cloud.google.com/kubernetes/config?project=gitlab-review-apps) or `kubectl get secret,cm --sort-by='{.metadata.creationTimestamp}' | grep 'review-'`. -Any secrets or config maps older than 5 days are suspect and should be deleted. +Any secrets or configuration maps older than 5 days are suspect and should be deleted. **Useful commands:** @@ -321,7 +321,7 @@ kubectl get cm --sort-by='{.metadata.creationTimestamp}' | grep 'review-' | grep #### Finding the problem -[In the past](https://gitlab.com/gitlab-org/gitlab-foss/issues/62834), it happened +[In the past](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62834), it happened that the `dns-gitlab-review-app-external-dns` Deployment was in a pending state, effectively preventing all the Review Apps from getting a DNS record assigned, making them unreachable via domain name. @@ -354,7 +354,7 @@ For the record, the debugging steps to find out this issue were: 1. Web search for exact error message, following rabbit hole to [a relevant Kubernetes bug report](https://github.com/kubernetes/kubernetes/issues/57345) 1. Access the node over SSH via the GCP console (**Computer Engine > VM instances** then click the "SSH" button for the node where the `dns-gitlab-review-app-external-dns` pod runs) -1. In the node: `systemctl --version` => systemd 232 +1. In the node: `systemctl --version` => `systemd 232` 1. Gather some more information: - `mount | grep kube | wc -l` => e.g. 290 - `systemctl list-units --all | grep -i var-lib-kube | wc -l` => e.g. 142 @@ -406,7 +406,7 @@ find a way to limit it to only us.** ## Other resources - [Review Apps integration for CE/EE (presentation)](https://docs.google.com/presentation/d/1QPLr6FO4LduROU8pQIPkX1yfGvD13GEJIBOenqoKxR8/edit?usp=sharing) -- [Stability issues](https://gitlab.com/gitlab-org/quality/team-tasks/issues/212) +- [Stability issues](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/212) ### Helpful command line tools diff --git a/doc/development/testing_guide/testing_levels.md b/doc/development/testing_guide/testing_levels.md index 9285a910ecf..2210ec94696 100644 --- a/doc/development/testing_guide/testing_levels.md +++ b/doc/development/testing_guide/testing_levels.md @@ -295,7 +295,7 @@ graph RL - **DOM**: Testing on the real DOM ensures your components work in the intended environment. - Part of DOM testing is delegated to [cross-browser testing](https://gitlab.com/gitlab-org/quality/team-tasks/issues/45). + Part of DOM testing is delegated to [cross-browser testing](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/45). - **Properties or state of components**: On this level, all tests can only perform actions a user would do. For example: to change the state of a component, a click event would be fired. @@ -366,7 +366,7 @@ See also: - The [RSpec testing guidelines](../testing_guide/best_practices.md#rspec). - System / Feature tests in the [Testing Best Practices](best_practices.md#system--feature-tests). -- [Issue #26159](https://gitlab.com/gitlab-org/gitlab/issues/26159) which aims at combining those guidelines with this page. +- [Issue #26159](https://gitlab.com/gitlab-org/gitlab/-/issues/26159) which aims at combining those guidelines with this page. ```mermaid graph RL diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md index 8c71a27540d..797dfc676eb 100644 --- a/doc/development/understanding_explain_plans.md +++ b/doc/development/understanding_explain_plans.md @@ -198,8 +198,7 @@ There are quite a few different types of nodes, so we only cover some of the more common ones here. A full list of all the available nodes and their descriptions can be found in -the [PostgreSQL source file -"plannodes.h"](https://gitlab.com/postgres/postgres/blob/master/src/include/nodes/plannodes.h) +the [PostgreSQL source file `plannodes.h`](https://gitlab.com/postgres/postgres/blob/master/src/include/nodes/plannodes.h) ### Seq Scan @@ -315,25 +314,30 @@ the `Indexes:` section: ```sql Indexes: "users_pkey" PRIMARY KEY, btree (id) - "users_confirmation_token_key" UNIQUE CONSTRAINT, btree (confirmation_token) - "users_email_key" UNIQUE CONSTRAINT, btree (email) - "users_reset_password_token_key" UNIQUE CONSTRAINT, btree (reset_password_token) - "index_on_users_lower_email" btree (lower(email::text)) - "index_on_users_lower_username" btree (lower(username::text)) + "index_users_on_confirmation_token" UNIQUE, btree (confirmation_token) + "index_users_on_email" UNIQUE, btree (email) + "index_users_on_reset_password_token" UNIQUE, btree (reset_password_token) + "index_users_on_static_object_token" UNIQUE, btree (static_object_token) + "index_users_on_unlock_token" UNIQUE, btree (unlock_token) "index_on_users_name_lower" btree (lower(name::text)) + "index_users_on_accepted_term_id" btree (accepted_term_id) "index_users_on_admin" btree (admin) "index_users_on_created_at" btree (created_at) "index_users_on_email_trigram" gin (email gin_trgm_ops) "index_users_on_feed_token" btree (feed_token) - "index_users_on_ghost" btree (ghost) + "index_users_on_group_view" btree (group_view) "index_users_on_incoming_email_token" btree (incoming_email_token) + "index_users_on_managing_group_id" btree (managing_group_id) "index_users_on_name" btree (name) "index_users_on_name_trigram" gin (name gin_trgm_ops) + "index_users_on_public_email" btree (public_email) WHERE public_email::text <> ''::text "index_users_on_state" btree (state) - "index_users_on_state_and_internal_attrs" btree (state) WHERE ghost <> true AND support_bot <> true - "index_users_on_support_bot" btree (support_bot) + "index_users_on_state_and_user_type" btree (state, user_type) + "index_users_on_unconfirmed_email" btree (unconfirmed_email) WHERE unconfirmed_email IS NOT NULL + "index_users_on_user_type" btree (user_type) "index_users_on_username" btree (username) "index_users_on_username_trigram" gin (username gin_trgm_ops) + "tmp_idx_on_user_id_where_bio_is_filled" btree (id) WHERE COALESCE(bio, ''::character varying)::text IS DISTINCT FROM ''::text ``` Here we can see there is no index on the `twitter` column, which means @@ -652,7 +656,7 @@ different queries. The only _rule_ is that you _must always measure_ your query and related tools such as: - [`explain.depesz.com`](https://explain.depesz.com/). -- [Pev](http://tatiyants.com/postgres-query-plan-visualization/). +- [`explain.dalibo.com/`](https://explain.dalibo.com/). ## Producing query plans @@ -681,11 +685,11 @@ Planning time: 0.411 ms Execution time: 0.113 ms ``` -### Chatops +### ChatOps -[GitLab employees can also use our chatops solution, available in Slack using the +[GitLab employees can also use our ChatOps solution, available in Slack using the `/chatops` slash command](chatops_on_gitlabcom.md). -You can use chatops to get a query plan by running the following: +You can use ChatOps to get a query plan by running the following: ```sql /chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) @@ -714,7 +718,7 @@ with their own clone of the production database. Joe is available in the [`#database-lab`](https://gitlab.slack.com/archives/CLJMDRD8C) channel on Slack. -Unlike chatops, it gives you a way to execute DDL statements (like creating indexes and tables) and get query plan not only for `SELECT` but also `UPDATE` and `DELETE`. +Unlike ChatOps, it gives you a way to execute DDL statements (like creating indexes and tables) and get query plan not only for `SELECT` but also `UPDATE` and `DELETE`. For example, in order to test new index you can do the following: diff --git a/doc/development/uploads.md b/doc/development/uploads.md index 1dd6ab5496a..4693c93e3d0 100644 --- a/doc/development/uploads.md +++ b/doc/development/uploads.md @@ -92,7 +92,7 @@ We can identify three major use-cases for an upload: 1. **storage:** if we are uploading for storing a file (i.e. artifacts, packages, discussion attachments). In this case [direct upload](#direct-upload) is the proper level as it's the less resource-intensive operation. Additional information can be found on [File Storage in GitLab](file_storage.md). 1. **in-controller/synchronous processing:** if we allow processing **small files** synchronously, using [disk buffered upload](#disk-buffered-upload) may speed up development. -1. **Sidekiq/asynchronous processing:** Async processing must implement [direct upload](#direct-upload), the reason being that it's the only way to support Cloud Native deployments without a shared NFS. +1. **Sidekiq/asynchronous processing:** Asynchronous processing must implement [direct upload](#direct-upload), the reason being that it's the only way to support Cloud Native deployments without a shared NFS. For more details about currently broken feature see [epic &1802](https://gitlab.com/groups/gitlab-org/-/epics/1802). @@ -114,7 +114,7 @@ We have three kinds of file encoding in our uploads: 1. <i class="fa fa-check-circle"></i> **multipart**: `multipart/form-data` is the most common, a file is encoded as a part of a multipart encoded request. 1. <i class="fa fa-check-circle"></i> **body**: some APIs uploads files as the whole request body. -1. <i class="fa fa-times-circle"></i> **JSON**: some JSON API uploads files as base64 encoded strings. This will require a change to GitLab Workhorse, which [is planned](https://gitlab.com/gitlab-org/gitlab-workhorse/issues/226). +1. <i class="fa fa-times-circle"></i> **JSON**: some JSON API uploads files as base64 encoded strings. This will require a change to GitLab Workhorse, which [is planned](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/226). ## Uploading technologies @@ -128,7 +128,7 @@ This is the default kind of upload, and it's most expensive in terms of resource In this case, workhorse is unaware of files being uploaded and acts as a regular proxy. -When a multipart request reaches the rails application, `Rack::Multipart` leaves behind tempfiles in `/tmp` and uses valuable Ruby process time to copy files around. +When a multipart request reaches the rails application, `Rack::Multipart` leaves behind temporary files in `/tmp` and uses valuable Ruby process time to copy files around. ```mermaid sequenceDiagram diff --git a/doc/development/utilities.md b/doc/development/utilities.md index dfdc5c66114..52151d37038 100644 --- a/doc/development/utilities.md +++ b/doc/development/utilities.md @@ -49,7 +49,7 @@ Refer to: <https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/utils/mer ## `Override` -Refer to <https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/utils/override.rb>: +Refer to [`override.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/utils/override.rb): - This utility can help you check if one method would override another or not. It is the same concept as Java's `@Override` annotation @@ -153,7 +153,7 @@ Refer to <https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/utils/stro ## `RequestCache` -Refer to <https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/cache/request_cache.rb>. +Refer to [`request_cache.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/cache/request_cache.rb). This module provides a simple way to cache values in RequestStore, and the cache key would be based on the class name, method name, diff --git a/doc/development/value_stream_analytics.md b/doc/development/value_stream_analytics.md index 69b07eb7c86..70b16cc1739 100644 --- a/doc/development/value_stream_analytics.md +++ b/doc/development/value_stream_analytics.md @@ -183,7 +183,7 @@ Currently supported parents: ### Default stages -The [original implementation](https://gitlab.com/gitlab-org/gitlab/issues/847) of value stream analytics defined 7 stages. These stages are always available for each parent, however altering these stages is not possible. +The [original implementation](https://gitlab.com/gitlab-org/gitlab/-/issues/847) of value stream analytics defined 7 stages. These stages are always available for each parent, however altering these stages is not possible. To make things efficient and reduce the number of records created, the default stages are expressed as in-memory objects (not persisted). When the user creates a custom stage for the first time, all the stages will be persisted. This behavior is implemented in the value stream analytics service objects. diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index 8ea9f70fc7a..407899b23d5 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -130,12 +130,12 @@ class CleanupUsersUpdatedAtRename < ActiveRecord::Migration[4.2] end ``` -NOTE: **Note:** If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/migration_helpers.rb#L9), please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet. +NOTE: **Note:** If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet. With [Canary](https://about.gitlab.com/handbook/engineering/infrastructure/library/canary/) it is possible that the system runs in this state for a significant amount of time. ## Changing Column Constraints -Adding or removing a NOT NULL clause (or another constraint) can typically be +Adding or removing a `NOT NULL` clause (or another constraint) can typically be done without requiring downtime. However, this does require that any application changes are deployed _first_. Thus, changing the constraints of a column should happen in a post-deployment migration. @@ -143,35 +143,11 @@ happen in a post-deployment migration. NOTE: Avoid using `change_column` as it produces an inefficient query because it re-defines the whole column type. -To add a NOT NULL constraint, use the `add_not_null_constraint` migration helper: +You can check the following guides for each specific use case: -```ruby -# A post-deployment migration in db/post_migrate -class AddNotNull < ActiveRecord::Migration[4.2] - include Gitlab::Database::MigrationHelpers - - disable_ddl_transaction! - - def up - add_not_null_constraint :users, :username - end - - def down - remove_not_null_constraint :users, :username - end -end -``` - -If the column to be updated requires cleaning first (e.g. there are `NULL` values), you should: - -1. Add the `NOT NULL` constraint with `validate: false` - - `add_not_null_constraint :users, :username, validate: false` - -1. Clean up the data with a data migration -1. Validate the `NOT NULL` constraint with a followup migration - - `validate_not_null_constraint :users, :username` +- [Adding foreign-key constraints](migration_style_guide.md#adding-foreign-key-constraints) +- [Adding `NOT NULL` constraints](database/not_null_constraints.md) +- [Adding limits to text columns](database/strings_and_the_text_data_type.md) ## Changing Column Types diff --git a/doc/development/windows.md b/doc/development/windows.md index b5309002222..c92a468fad3 100644 --- a/doc/development/windows.md +++ b/doc/development/windows.md @@ -47,7 +47,7 @@ There is a chance that your Google Cloud group may already have an image built. Search the available images before you do the work to build your own. -Build a Google Cloud image with the above shared runners repo by doing the following: +Build a Google Cloud image with the above shared runners repository by doing the following: 1. Install [Packer](https://www.packer.io/) (tested to work with version 1.5.1). 1. Install Packer Windows Update Provisioner. @@ -55,7 +55,7 @@ Build a Google Cloud image with the above shared runners repo by doing the follo 1. Run the command `go build -o packer-provisioner-windows-update` (requires `go` to be installed). 1. Verify `packer-provisioner-windows-update` is in the `PATH` environment variable. 1. Add all [required environment variables](https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/gcp/windows-containers/-/blob/master/packer.json#L2-10) - in the `packer.json` file to your environment (perhaps use [direnv](https://direnv.net/)). + in the `packer.json` file to your environment (perhaps use [`direnv`](https://direnv.net/)). 1. Build the image by running the command: `packer build packer.json`. ## How to use a Windows image in GCP @@ -96,7 +96,7 @@ Here are a few tips on GCP and Windows. ### GCP cost savings To minimise the cost of your GCP VM instance, stop it when you're not using it. -If you do, you'll need to redownload the RDP file from the console as the IP +If you do, you'll need to re-download the RDP file from the console as the IP address changes every time you stop and start it. ### chocolatey @@ -119,13 +119,13 @@ You can install .NET version 3 support with the following `DISM` command: `DISM /Online /Enable-Feature /FeatureName:NetFx3 /All` -### nix -> Windows cmd tips +### nix -> Windows `cmd` tips -The first tip for using the Windows command shell is to open Powershell and use that instead. +The first tip for using the Windows command shell is to open PowerShell and use that instead. -Start Powershell: `start powershell`. +Start PowerShell: `start powershell`. -Powershell has aliases for all of the following commands so you don't have to learn the native commands: +PowerShell has aliases for all of the following commands so you don't have to learn the native commands: - `ls` ---> `dir` - `rm` ---> `del` diff --git a/doc/gitlab-basics/create-project.md b/doc/gitlab-basics/create-project.md index 152fb24699e..def14cfce6d 100644 --- a/doc/gitlab-basics/create-project.md +++ b/doc/gitlab-basics/create-project.md @@ -103,7 +103,7 @@ You can improve the existing built-in templates or contribute new ones in the #### Custom project templates **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6860) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6860) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. Creating new projects based on custom project templates is a convenient option for quickly starting projects. @@ -123,7 +123,7 @@ To use a custom project template on the **New project** page: ## Push to create a new project -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/26388) in GitLab 10.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/26388) in GitLab 10.5. When you create a new repository locally, instead of going to GitLab to manually create a new project and then [clone the repo](start-using-git.md#clone-a-repository) diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 9ebcf258ee9..439032f39b3 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -22,6 +22,11 @@ This guide will help you get started with Git through the command line and can b for Git commands in the future. If you're only looking for a quick reference of Git commands, you can download GitLab's [Git Cheat Sheet](https://about.gitlab.com/images/press/git-cheat-sheet.pdf). +> For more information about the advantages of working with Git and GitLab: +> +> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch the [GitLab Source Code Management Walkthrough](https://www.youtube.com/watch?v=wTQ3aXJswtM) video. +> - Learn how GitLab became the backbone of [Worldline](https://about.gitlab.com/customers/worldline/)’s development environment. + TIP: **Tip:** To help you visualize what you're doing locally, there are [Git GUI apps](https://git-scm.com/download/gui/) you can install. @@ -45,7 +50,7 @@ prompt, terminal, and command line) of your preference. Here are some suggestion - [iTerm2](https://www.iterm2.com/), which you can integrate with [zsh](https://git-scm.com/book/id/v2/Appendix-A%3A-Git-in-Other-Environments-Git-in-Zsh) and [oh my zsh](https://ohmyz.sh/) for color highlighting, among other handy features for Git users. - For Windows users: - Built-in: **cmd**. Click the search icon on the bottom navbar on Windows and type "cmd" to find it. - - [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-windows-powershell?view=powershell-7): a Windows "powered up" shell, from which you can execute a greater number of commands. + - [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/windows-powershell/install/installing-windows-powershell?view=powershell-7): a Windows "powered up" shell, from which you can execute a greater number of commands. - Git Bash: it comes built into [Git for Windows](https://gitforwindows.org/). - For Linux users: - Built-in: [Linux Terminal](https://www.howtogeek.com/140679/beginner-geek-how-to-start-using-the-linux-terminal/). diff --git a/doc/install/aws/img/aws_ha_architecture_diagram.png b/doc/install/aws/img/aws_ha_architecture_diagram.png Binary files differindex e019ed61abf..dc63d36e0b3 100644 --- a/doc/install/aws/img/aws_ha_architecture_diagram.png +++ b/doc/install/aws/img/aws_ha_architecture_diagram.png diff --git a/doc/install/aws/img/rds_subnet_group.png b/doc/install/aws/img/rds_subnet_group.png Binary files differdeleted file mode 100644 index 7c6157e38e0..00000000000 --- a/doc/install/aws/img/rds_subnet_group.png +++ /dev/null diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 41f8d7babac..813e343f2cc 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -12,9 +12,7 @@ For organizations with 300 users or less, the recommended AWS installation metho ## Introduction -GitLab on AWS can leverage many of the services that are already -configurable. These services offer a great deal of -flexibility and can be adapted to the needs of most companies. +For the most part, we'll make use of Omnibus GitLab in our setup, but we'll also leverage native AWS services. Instead of using the Omnibus bundled PostgreSQL and Redis, we will use AWS RDS and ElastiCache. In this guide, we'll go through a multi-node setup where we'll start by configuring our Virtual Private Cloud and subnets to later integrate @@ -59,8 +57,6 @@ Here's a list of the AWS services we will use, with links to pricing information Redis configuration. See the [Amazon ElastiCache pricing](https://aws.amazon.com/elasticache/pricing/). -NOTE: **Note:** Please note that while we will be using EBS for storage, we do not recommend using EFS as it may negatively impact GitLab's performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. - ## Create an IAM EC2 instance role and profile As we'll be using [Amazon S3 object storage](#amazon-s3-object-storage), our EC2 instances need to have read, write, and list permissions for our S3 buckets. To avoid embedding AWS keys in our GitLab config, we'll make use of an [IAM Role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) to allow our GitLab instance with this access. We'll need to create an IAM policy to attach to our IAM role: @@ -227,17 +223,21 @@ We also need to create two private route tables so that instances in each privat ## Load Balancer +We'll create a load balancer to evenly distribute inbound traffic on ports `80` and `443` across our GitLab application servers. Based the on the [scaling policies](#create-an-auto-scaling-group) we'll create later, instances will be added to or removed from our load balancer as needed. Additionally, the load balance will perform health checks on our instances. + On the EC2 dashboard, look for Load Balancer in the left navigation bar: 1. Click the **Create Load Balancer** button. 1. Choose the **Classic Load Balancer**. 1. Give it a name (we'll use `gitlab-loadbalancer`) and for the **Create LB Inside** option, select `gitlab-vpc` from the dropdown menu. 1. In the **Listeners** section, set HTTP port 80, HTTPS port 443, and TCP port 22 for both load balancer and instance protocols and ports. - 1. In the **Select Subnets** section, select both public subnets from the list. -1. Click **Assign Security Groups** and select **Create a new security group**, give it a name + 1. In the **Select Subnets** section, select both public subnets from the list so that the load balancer can route traffic to both availability zones. +1. We'll add a security group for our load balancer to act as a firewall to control what traffic is allowed through. Click **Assign Security Groups** and select **Create a new security group**, give it a name (we'll use `gitlab-loadbalancer-sec-group`) and description, and allow both HTTP and HTTPS traffic - from anywhere (`0.0.0.0/0, ::/0`). Also allow SSH traffic from a single IP address or an IP address range in CIDR notation. -1. Click **Configure Security Settings** and select an SSL/TLS certificate from ACM or upload a certificate to IAM. + from anywhere (`0.0.0.0/0, ::/0`). Also allow SSH traffic, select a custom source, and add a single trusted IP address or an IP address range in CIDR notation. This will allow users to perform Git actions over SSH. +1. Click **Configure Security Settings** and set the following: + 1. Select an SSL/TLS certificate from ACM or upload a certificate to IAM. + 1. Under **Select a Cipher**, pick a predefined security policy from the dropdown. You can see a breakdown of [Predefined SSL Security Policies for Classic Load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-security-policy-table.html) in the AWS docs. Check the GitLab codebase for a list of [supported SSL ciphers and protocols](https://gitlab.com/gitlab-org/gitlab/-/blob/9ee7ad433269b37251e0dd5b5e00a0f00d8126b4/lib/support/nginx/gitlab-ssl#L97-99). 1. Click **Configure Health Check** and set up a health check for your EC2 instances. 1. For **Ping Protocol**, select HTTP. 1. For **Ping Port**, enter 80. @@ -261,11 +261,16 @@ On the Route 53 dashboard, click **Hosted zones** in the left navigation bar: 1. Click **Create Record Set** and provide the following values: 1. **Name:** Use the domain name (the default value) or enter a subdomain. 1. **Type:** Select **A - IPv4 address**. + 1. **Alias:** Defaults to **No**. Select **Yes**. 1. **Alias Target:** Find the **ELB Classic Load Balancers** section and select the classic load balancer we created earlier. 1. **Routing Policy:** We'll use **Simple** but you can choose a different policy based on your use case. 1. **Evaluate Target Health:** We'll set this to **No** but you can choose to have the load balancer route traffic based on target health. 1. Click **Create**. -1. Update your DNS records with your domain registrar. The steps for doing this vary depending on which registrar you use and is beyond the scope of this guide. +1. If you registered your domain through Route 53, you're done. If you used a different domain registrar, you need to update your DNS records with your domain registrar. You'll need to: + 1. Click on **Hosted zones** and select the domain you added above. + 1. You'll see a list of `NS` records. From your domain registrar's admin panel, add each of these as `NS` records to your domain's DNS records. These steps may vary between domain registrars. If you're stuck, Google **"name of your registrar" add dns records** and you should find a help article specific to your domain registrar. + +The steps for doing this vary depending on which registrar you use and is beyond the scope of this guide. ## PostgreSQL with RDS @@ -280,7 +285,10 @@ We need a security group for our database that will allow inbound traffic from t 1. From the EC2 dashboard, select **Security Groups** from the left menu bar. 1. Click **Create security group**. 1. Give it a name (we'll use `gitlab-rds-sec-group`), a description, and select the `gitlab-vpc` from the **VPC** dropdown. -1. In the **Inbound rules** section, click **Add rule** and add a **PostgreSQL** rule, and set the "Custom" source as the `gitlab-loadbalancer-sec-group` we created earlier. The default PostgreSQL port is `5432`, which we'll also use when creating our database below. +1. In the **Inbound rules** section, click **Add rule** and set the following: + 1. **Type:** search for and select the **PostgreSQL** rule. + 1. **Source type:** set as "Custom". + 1. **Source:** select the `gitlab-loadbalancer-sec-group` we created earlier. 1. When done, click **Create security group**. ### RDS Subnet Group @@ -288,11 +296,10 @@ We need a security group for our database that will allow inbound traffic from t 1. Navigate to the RDS dashboard and select **Subnet Groups** from the left menu. 1. Click on **Create DB Subnet Group**. 1. Under **Subnet group details**, enter a name (we'll use `gitlab-rds-group`), a description, and choose the `gitlab-vpc` from the VPC dropdown. -1. Under **Add subnets**, click **Add all the subnets related to this VPC** and remove the public ones, we only want the **private subnets**. In the end, you should see `10.0.1.0/24` and `10.0.3.0/24` (as we defined them in the [subnets section](#subnets)). +1. From the **Availability Zones** dropdown, select the Availability Zones that include the subnets you've configured. In our case, we'll add `eu-west-2a` and `eu-west-2b`. +1. From the **Subnets** dropdown, select the two private subnets (`10.0.1.0/24` and `10.0.3.0/24`) as we defined them in the [subnets section](#subnets). 1. Click **Create** when ready. -  - ### Create the database DANGER: **Danger:** Avoid using burstable instances (t class instances) for the database as this could lead to performance issues due to CPU credits running out during sustained periods of high load. @@ -301,7 +308,7 @@ Now, it's time to create the database: 1. Navigate to the RDS dashboard, select **Databases** from the left menu, and click **Create database**. 1. Select **Standard Create** for the database creation method. -1. Select **PostgreSQL** as the database engine and select **PostgreSQL 10.9-R1** from the version dropdown menu (check the [database requirements](../../install/requirements.md#postgresql-requirements) to see if there are any updates on this for your chosen version of GitLab). +1. Select **PostgreSQL** as the database engine and select the minimum PostgreSQL version as defined for your GitLab version in our [database requirements](../../install/requirements.md#postgresql-requirements). 1. Since this is a production server, let's choose **Production** from the **Templates** section. 1. Under **Settings**, set a DB instance identifier, a master username, and a master password. We'll use `gitlab-db-ha`, `gitlab`, and a very secure password respectively. Make a note of these as we'll need them later. 1. For the DB instance size, select **Standard classes** and select an instance size that meets your requirements from the dropdown menu. We'll use a `db.m4.large` instance. @@ -329,7 +336,7 @@ Now that the database is created, let's move on to setting up Redis with ElastiC ## Redis with ElastiCache ElastiCache is an in-memory hosted caching solution. Redis maintains its own -persistence and is used for certain types of the GitLab application. +persistence and is used to store session data, temporary cache information, and background job queues for the GitLab application. ### Create a Redis Security Group @@ -559,9 +566,9 @@ Let's create an EC2 instance where we'll install Gitaly: 1. Click **Review and launch** followed by **Launch** if you're happy with your settings. 1. Finally, acknowledge that you have access to the selected private key file or create a new one. Click **Launch Instances**. - > **Optional:** Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). +NOTE: **Optional:** Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). We do not recommend using EFS as it may negatively impact GitLab’s performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. -Now that we have our EC2 instance ready, follow the [documentation to install GitLab and set up Gitaly on its own server](../../administration/gitaly/index.md#running-gitaly-on-its-own-server). Perform the client setup steps from that document on the [GitLab instance we created](#install-gitlab) above. +Now that we have our EC2 instance ready, follow the [documentation to install GitLab and set up Gitaly on its own server](../../administration/gitaly/index.md#run-gitaly-on-its-own-server). Perform the client setup steps from that document on the [GitLab instance we created](#install-gitlab) above. #### Add Support for Proxied SSL @@ -600,7 +607,7 @@ sudo cp -R /etc/ssh/* /etc/ssh_static In `/etc/ssh/sshd_config` update the following: -```bash +```shell # HostKeys for protocol version 2 HostKey /etc/ssh_static/ssh_host_rsa_key HostKey /etc/ssh_static/ssh_host_dsa_key @@ -620,7 +627,7 @@ Remember to run `sudo gitlab-ctl reconfigure` after saving the changes to the `g NOTE: **Note:** One current feature of GitLab that still requires a shared directory (NFS) is [GitLab Pages](../../user/project/pages/index.md). -There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) +There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196) to eliminate the need for NFS to support GitLab Pages. --- @@ -774,7 +781,7 @@ And the more complex the solution, the more work is involved in setting up and maintaining it. Have a read through these other resources and feel free to -[open an issue](https://gitlab.com/gitlab-org/gitlab/issues/new) +[open an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) to request additional material: - [Scaling GitLab](../../administration/reference_architectures/index.md): diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index fbc81da20d4..3cf963bdf57 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -177,7 +177,7 @@ Click **"Save"** for the changes to take effect. domain registrar which points to the public IP address of your Azure VM. If you do this, you'll need to make sure your VM is configured to use a _static_ public IP address (i.e. not a _dynamic_ one) or you will have to reconfigure the DNS `A` record each time Azure reassigns your VM a new public IP -address. Read [IP address types and allocation methods in Azure](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-ip-addresses-overview-arm) to learn more. +address. Read [Public IP addresses](https://docs.microsoft.com/en-us/azure/virtual-network/public-ip-addresses) to learn more. ## Let's open some ports diff --git a/doc/install/digitaloceandocker.md b/doc/install/digitaloceandocker.md index ccfb97afe28..fe32b37a9ed 100644 --- a/doc/install/digitaloceandocker.md +++ b/doc/install/digitaloceandocker.md @@ -31,12 +31,12 @@ locally on either macOS or Linux. NOTE: **Note:** The rest of the steps are identical for macOS and Linux. -## Create new docker host +## Create new Docker host 1. Login to Digital Ocean. 1. Generate a new API token at <https://cloud.digitalocean.com/settings/api/tokens>. - This command will create a new DO droplet called `gitlab-test-env-do` that will act as a docker host. + This command will create a new DO droplet called `gitlab-test-env-do` that will act as a Docker host. NOTE: **Note:** 4GB is the minimum requirement for a Docker host that will run more than one GitLab instance. @@ -69,20 +69,20 @@ Resource: <https://docs.docker.com/machine/drivers/digital-ocean/>. In this example we'll create a GitLab EE 8.10.8 instance. -First connect the docker client to the docker host you created previously. +First connect the Docker client to the Docker host you created previously. ```shell eval "$(docker-machine env gitlab-test-env-do)" ``` -You can add this to your `~/.bash_profile` file to ensure the `docker` client uses the `gitlab-test-env-do` docker host +You can add this to your `~/.bash_profile` file to ensure the `docker` client uses the `gitlab-test-env-do` Docker host ### Create new GitLab container - HTTP port: `8888` - SSH port: `2222` - Set `gitlab_shell_ssh_port` using `--env GITLAB_OMNIBUS_CONFIG` -- Hostname: IP of docker host +- Hostname: IP of Docker host - Container name: `gitlab-test-8.10` - GitLab version: **EE** `8.10.8-ee.0` @@ -108,7 +108,7 @@ gitlab/gitlab-ee:$VERSION ### Connect to the GitLab container -#### Retrieve the docker host IP +#### Retrieve the Docker host IP ```shell docker-machine ip gitlab-test-env-do diff --git a/doc/install/google_cloud_platform/img/boot_disk.png b/doc/install/google_cloud_platform/img/boot_disk.png Binary files differdeleted file mode 100644 index b9f7eed6601..00000000000 --- a/doc/install/google_cloud_platform/img/boot_disk.png +++ /dev/null diff --git a/doc/install/google_cloud_platform/img/vm_details.png b/doc/install/google_cloud_platform/img/vm_details.png Binary files differindex 85b9ca066c8..aab9a46fa11 100644 --- a/doc/install/google_cloud_platform/img/vm_details.png +++ b/doc/install/google_cloud_platform/img/vm_details.png diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index 433eeda00b1..8ca5c5c266a 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -33,13 +33,13 @@ To deploy GitLab on GCP you first need to create a virtual machine:  1. On the next page, you can select the type of VM as well as the - estimated costs. Provide the name of the instance, desired datacenter, and machine type. Note that GitLab recommends at least 2 vCPU's and 4GB of RAM. + estimated costs. Provide the name of the instance, desired datacenter, and machine type. + Note our [hardware requirements for different user base sizes](../requirements.md#hardware-requirements).  -1. Click **Change** under Boot disk to select the size, type, and desired operating system. GitLab supports a [variety of linux operating systems](../requirements.md), including Ubuntu and Debian. Click **Select** when finished. - -  +1. To select the size, type, and desired [operating system](../requirements.md#supported-linux-distributions), + click **Change** under `Boot disk`. Click **Select** when finished. 1. As a last step allow HTTP and HTTPS traffic, then click **Create**. The process will finish in a few seconds. diff --git a/doc/install/installation.md b/doc/install/installation.md index dd619e9e7b3..0997062006d 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -134,7 +134,7 @@ Make sure you have the right version of Git installed: # Install Git sudo apt-get install -y git-core -# Make sure Git is version 2.26.2 or higher (minimal supported version is 2.22.0) +# Make sure Git is version 2.27.0 or higher (minimal supported version is 2.25.0) git --version ``` @@ -142,7 +142,7 @@ Starting with GitLab 12.0, Git is required to be compiled with `libpcre2`. Find out if that's the case: ```shell -ldd $(which git) | grep pcre2 +ldd $(command -v git) | grep pcre2 ``` The output should contain `libpcre2-8.so.0`. @@ -181,9 +181,9 @@ sudo make install # Download and compile from source cd /tmp -curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.26.2.tar.gz -echo 'e1c17777528f55696815ef33587b1d20f5eec246669f3b839d15dbfffad9c121 git-2.26.2.tar.gz' | shasum -a256 -c - && tar -xzf git-2.26.2.tar.gz -cd git-2.26.2/ +curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.27.0.tar.gz +echo '77ded85cbe42b1ffdc2578b460a1ef5d23bcbc6683eabcafbb0d394dffe2e787 git-2.27.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.27.0.tar.gz +cd git-2.27.0/ ./configure --with-libpcre make prefix=/usr/local all @@ -200,7 +200,7 @@ needs to be installed. sudo apt-get install -y graphicsmagick ``` -**Note:** In order to receive mail notifications, make sure to install a mail server. By default, Debian is shipped with exim4 but this [has problems](https://gitlab.com/gitlab-org/gitlab-foss/issues/12754) while Ubuntu does not ship with one. The recommended mail server is postfix and you can install it with: +**Note:** In order to receive mail notifications, make sure to install a mail server. By default, Debian is shipped with exim4 but this [has problems](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/12754) while Ubuntu does not ship with one. The recommended mail server is postfix and you can install it with: ```shell sudo apt-get install -y postfix @@ -208,6 +208,13 @@ sudo apt-get install -y postfix Then select 'Internet Site' and press enter to confirm the hostname. +[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse#dependencies) +requires `exiftool` to remove EXIF data from uploaded images. + +```shell +sudo apt-get install -y libimage-exiftool-perl +``` + ## 2. Ruby The Ruby interpreter is required to run GitLab. @@ -216,10 +223,9 @@ The Ruby interpreter is required to run GitLab. dropped support for Ruby 2.5.x. The use of Ruby version managers such as [RVM](https://rvm.io/), [rbenv](https://github.com/rbenv/rbenv) or [chruby](https://github.com/postmodern/chruby) with GitLab -in production, frequently leads to hard to diagnose problems. For example, -GitLab Shell is called from OpenSSH, and having a version manager can prevent -pushing and pulling over SSH. Version managers are not supported and we strongly -advise everyone to follow the instructions below to use a system Ruby. +in production, frequently leads to hard to diagnose problems. Version managers +are not supported and we strongly advise everyone to follow the instructions +below to use a system Ruby. Linux distributions generally have older versions of Ruby available, so these instructions are designed to install Ruby from the official source code. @@ -234,9 +240,9 @@ Download Ruby and compile it: ```shell mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.gz -echo '1416ce288fb8bfeae07a12b608540318c9cace71 ruby-2.6.5.tar.gz' | shasum -c - && tar xzf ruby-2.6.5.tar.gz -cd ruby-2.6.5 +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.6.tar.gz +echo '2d78048e293817f38d4ede4ebc7873013e97bb0b ruby-2.6.6.tar.gz' | shasum -c - && tar xzf ruby-2.6.6.tar.gz +cd ruby-2.6.6 ./configure --disable-install-rdoc make @@ -565,8 +571,8 @@ If you want to use Kerberos for user authentication, omit `kerberos` in the `--w GitLab Shell is an SSH access and repository management software developed specially for GitLab. ```shell -# Run the installation task for gitlab-shell (replace `REDIS_URL` if needed): -sudo -u git -H bundle exec rake gitlab:shell:install REDIS_URL=unix:/var/run/redis/redis.sock RAILS_ENV=production SKIP_STORAGE_VALIDATION=true +# Run the installation task for gitlab-shell: +sudo -u git -H bundle exec rake gitlab:shell:install RAILS_ENV=production # By default, the gitlab-shell config is generated from your main GitLab config. # You can review (and modify) the gitlab-shell config as follows: @@ -579,13 +585,6 @@ If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional ste NOTE: **Note:** Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in `/etc/hosts` ("127.0.0.1 hostname"). This might be necessary, for example, if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check will fail with `Check GitLab API access: FAILED. code: 401` and pushing commits will be rejected with `[remote rejected] master -> master (hook declined)`. -NOTE: **Note:** -GitLab Shell application startup time can be greatly reduced by disabling RubyGems. This can be done in several ways: - -- Export `RUBYOPT=--disable-gems` environment variable for the processes. -- Compile Ruby with `configure --disable-rubygems` to disable RubyGems by default. Not recommended for system-wide Ruby. -- Omnibus GitLab [replaces the *shebang* line of the `gitlab-shell/bin/*` scripts](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1707). - ### Install GitLab Workhorse GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The @@ -986,7 +985,7 @@ Using Sidekiq directly will still be supported until 14.0. So if you're experien 1. Edit the system `init.d` script to remove the `SIDEKIQ_WORKERS` flag. If you have `/etc/default/gitlab`, then you should edit it instead. 1. Restart GitLab. -1. [Create an issue](https://gitlab.com/gitlab-org/gitlab/issues/-/new) describing the problem. +1. [Create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/-/new) describing the problem. ## Troubleshooting diff --git a/doc/install/ldap.md b/doc/install/ldap.md index d8d54864586..164478f09f7 100644 --- a/doc/install/ldap.md +++ b/doc/install/ldap.md @@ -1,5 +1,5 @@ --- -redirect_to: '../administration/auth/ldap.md' +redirect_to: '../administration/auth/ldap/index.md' --- -This document was moved to [another location](../administration/auth/ldap.md). +This document was moved to [another location](../administration/auth/ldap/index.md). diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index fd81b7f6caf..7ae23d6831e 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -253,7 +253,7 @@ related object definitions to be created together, as well as a set of parameters for those objects. The template for GitLab resides in the Omnibus GitLab repository under the -docker directory. Let's download it locally with `wget`: +Docker directory. Let's download it locally with `wget`: ```shell wget https://gitlab.com/gitlab-org/omnibus-gitlab/raw/master/docker/openshift-template.json @@ -324,7 +324,7 @@ Now that we configured this, let's see how to manage and scale GitLab. Setting up GitLab for the first time might take a while depending on your internet connection and the resources you have attached to the all-in-one VM. -GitLab's docker image is quite big (~500MB), so you'll have to wait until +GitLab's Docker image is quite big (~500MB), so you'll have to wait until it's downloaded and configured before you use it. ### Watch while GitLab gets deployed diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 09ad5f9afd7..0673f3e7ea3 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -35,7 +35,7 @@ Please see the [installation from source guide](installation.md) and the [instal ### Microsoft Windows GitLab is developed for Linux-based operating systems. -It does **not** run on Microsoft Windows, and we have no plans to support it in the near future. For the latest development status view this [issue](https://gitlab.com/gitlab-org/gitlab/issues/22337). +It does **not** run on Microsoft Windows, and we have no plans to support it in the near future. For the latest development status view this [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/22337). Please consider using a virtual machine to run GitLab. ## Software requirements @@ -45,29 +45,29 @@ Please consider using a virtual machine to run GitLab. GitLab requires Ruby (MRI) 2.6. Beginning in GitLab 12.2, we no longer support Ruby 2.5 and lower. You must use the standard MRI implementation of Ruby. -We love [JRuby](https://www.jruby.org/) and [Rubinius](https://rubinius.com), but GitLab +We love [JRuby](https://www.jruby.org/) and [Rubinius](https://github.com/rubinius/rubinius#the-rubinius-language-platform), but GitLab needs several Gems that have native extensions. ### Go versions -The minimum required Go version is 1.12. +The minimum required Go version is 1.13. ### Git versions -GitLab 11.11 and higher only supports Git 2.21.x and newer, and -[dropped support for older versions](https://gitlab.com/gitlab-org/gitlab-foss/issues/54255). +GitLab 11.11 and higher only supports Git 2.24.x and newer, and +[dropped support for older versions](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54255). ### Node.js versions Beginning in GitLab 12.9, we only support node.js 10.13.0 or higher, and we have dropped support for node.js 8. (node.js 6 support was dropped in GitLab 11.8) -We recommend Node 12.x, as it is faster. +We recommend Node 12.x, as it's faster. GitLab uses [webpack](https://webpack.js.org/) to compile frontend assets, which requires a minimum version of Node.js 10.13.0. -You can check which version you are running with `node -v`. If you are running +You can check which version you're running with `node -v`. If you're running a version older than `v10.13.0`, you need to update it to a newer version. You can find instructions to install from community maintained packages or compile from source at the [Node.js website](https://nodejs.org/en/download/). @@ -80,122 +80,82 @@ GitLab requires Redis 5.0+. Beginning in GitLab 13.0, lower versions are not sup ### Storage -The necessary hard drive space largely depends on the size of the repos you want to store in GitLab but as a *rule of thumb* you should have at least as much free space as all your repos combined take up. +The necessary hard drive space largely depends on the size of the repositories you want to store in GitLab but as a *rule of thumb* you should have at least as much free space as all your repositories combined take up. -If you want to be flexible about growing your hard drive space in the future consider mounting it using LVM so you can add more hard drives when you need them. +If you want to be flexible about growing your hard drive space in the future consider mounting it using [logical volume management (LVM)](https://en.wikipedia.org/wiki/Logical_volume_management) so you can add more hard drives when you need them. Apart from a local hard drive you can also mount a volume that supports the network file system (NFS) protocol. This volume might be located on a file server, a network attached storage (NAS) device, a storage area network (SAN) or on an Amazon Web Services (AWS) Elastic Block Store (EBS) volume. -If you have enough RAM memory and a recent CPU the speed of GitLab is mainly limited by hard drive seek times. Having a fast drive (7200 RPM and up) or a solid state drive (SSD) will improve the responsiveness of GitLab. +If you have enough RAM and a recent CPU the speed of GitLab is mainly limited by hard drive seek times. Having a fast drive (7200 RPM and up) or a solid state drive (SSD) will improve the responsiveness of GitLab. -NOTE: **Note:** Since file system performance may affect GitLab's overall performance, we do not recommend using EFS for storage. See the [relevant documentation](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. +NOTE: **Note:** Since file system performance may affect GitLab's overall performance, [we don't recommend using AWS EFS for storage](../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs). ### CPU -This is the recommended minimum hardware for a handful of example GitLab user base sizes. Your exact needs may be more, depending on your workload. Your workload is influenced by factors such as - but not limited to - how active your users are, how much automation you use, mirroring, and repo/change size. +CPU requirements are dependent on the number of users and expected workload. Your exact needs may be more, depending on your workload. Your workload is influenced by factors such as - but not limited to - how active your users are, how much automation you use, mirroring, and repo/change size. -- 1 core supports up to 100 users but the application can be a bit slower due to having all workers and background jobs running on the same core -- **2 cores** is the **recommended** minimum number of cores and supports up to 100 users -- 4 cores supports up to 500 users -- 8 cores supports up to 1,000 users -- 32 cores supports up to 5,000 users +The following is the recommended minimum CPU hardware guidance for a handful of example GitLab user base sizes. + +- **4 cores** is the **recommended** minimum number of cores and supports up to 500 users +- 8 cores supports up to 1000 users - More users? Consult the [reference architectures page](../administration/reference_architectures/index.md) ### Memory -This is the recommended minimum hardware for a handful of example GitLab user base sizes. Your exact needs may be more, depending on your workload. Your workload is influenced by factors such as - but not limited to - how active your users are, how much automation you use, mirroring, and repo/change size. +Memory requirements are dependent on the number of users and expected workload. Your exact needs may be more, depending on your workload. Your workload is influenced by factors such as - but not limited to - how active your users are, how much automation you use, mirroring, and repo/change size. -You need at least 8GB of addressable memory (RAM + swap) to install and use GitLab! -The operating system and any other running applications will also be using memory -so keep in mind that you need at least 4GB available before running GitLab. With -less memory GitLab will give strange errors during the reconfigure run and 500 -errors during usage. +The following is the recommended minimum Memory hardware guidance for a handful of example GitLab user base sizes. -- 4GB RAM + 4GB swap supports up to 100 users but it will be very slow -- **8GB RAM** is the **recommended** minimum memory size for all installations and supports up to 100 users -- 16GB RAM supports up to 500 users -- 32GB RAM supports up to 1,000 users -- 128GB RAM supports up to 5,000 users +- **4GB RAM** is the **required** minimum memory size and supports up to 500 users + - Our [Memory Team](https://about.gitlab.com/handbook/engineering/development/enablement/memory/) is working to reduce the memory requirement. +- 8GB RAM supports up to 1000 users - More users? Consult the [reference architectures page](../administration/reference_architectures/index.md) -We recommend having at least [2GB of swap on your server](https://askubuntu.com/a/505344/310789), even if you currently have -enough available RAM. Having swap will help reduce the chance of errors occurring -if your available memory changes. We also recommend [configuring the kernel's swappiness setting](https://askubuntu.com/a/103916) +In addition to the above, we generally recommend having at least 2GB of swap on your server, +even if you currently have enough available RAM. Having swap will help reduce the chance of errors occurring +if your available memory changes. We also recommend configuring the kernel's swappiness setting to a low value like `10` to make the most of your RAM while still having the swap available when needed. -Our [Memory Team](https://about.gitlab.com/handbook/engineering/development/enablement/memory/) is actively working to reduce the memory requirement. - -NOTE: **Note:** The 25 workers of Sidekiq will show up as separate processes in your process overview (such as `top` or `htop`) but they share the same RAM allocation since Sidekiq is a multithreaded application. Please see the section below about Unicorn workers for information about how many you need for those. - ## Database -The server running the database should have _at least_ 5-10 GB of storage -available, though the exact requirements depend on the size of the GitLab -installation (e.g. the number of users, projects, etc). - -We currently support the following databases: - -- PostgreSQL - +PostgreSQL is the only supported database, which is bundled with the Omnibus GitLab package. +You can also use an [external PostgreSQL database](https://docs.gitlab.com/omnibus/settings/database.html#using-a-non-packaged-postgresql-database-management-server). Support for MySQL was removed in GitLab 12.1. Existing users using GitLab with MySQL/MariaDB are advised to [migrate to PostgreSQL](../update/mysql_to_postgresql.md) before upgrading. ### PostgreSQL Requirements +The server running PostgreSQL should have _at least_ 5-10 GB of storage +available, though the exact requirements [depend on the number of users](../administration/reference_architectures/index.md). + We highly recommend users to use the minimum PostgreSQL versions specified below as these are the versions used for development and testing. GitLab version | Minimum PostgreSQL version -|- 10.0 | 9.6 12.10 | 11 +13.0 | 11 -Users using PostgreSQL must ensure the `pg_trgm` extension is loaded into every -GitLab database. This extension can be enabled (using a PostgreSQL super user) -by running the following query for every database: - -```sql -CREATE EXTENSION pg_trgm; -``` +You must also ensure the `pg_trgm` extension is loaded into every +GitLab database. This extension [can be enabled](https://www.postgresql.org/docs/11/sql-createextension.html) using a PostgreSQL super user. -On some systems you may need to install an additional package (e.g. +On some systems you may need to install an additional package (for example, `postgresql-contrib`) for this extension to become available. -NOTE: **Note:** Support for PostgreSQL 9.6 and 10 will be removed in GitLab 13.0 so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. For the schedule on adding support for PostgreSQL 11 and 12, see [the related epic](https://gitlab.com/groups/gitlab-org/-/epics/2184). For the release schedule for GitLab 13.0, see [GitLab's release and maintenance policy](../policy/maintenance.md). +NOTE: **Note:** Support for [PostgreSQL 9.6 and 10 has been removed in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab) so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. For the schedule of transitioning to PostgreSQL 12, see [the related epic](https://gitlab.com/groups/gitlab-org/-/epics/2184). #### Additional requirements for GitLab Geo -If you are using [GitLab Geo](../development/geo.md): +If you're using [GitLab Geo](../administration/geo/replication/index.md): - We strongly recommend running Omnibus-managed instances as they are actively developed and tested. We aim to be compatible with most external (not managed - by Omnibus) databases (for example, AWS RDS) but we do not guarantee - compatibility. -- The - [tracking database](../development/geo.md#using-the-tracking-database) - requires the - [postgres_fdw](https://www.postgresql.org/docs/11/postgres-fdw.html) - extension. - -```sql -CREATE EXTENSION postgres_fdw; -``` - -## Unicorn Workers - -For most instances we recommend using: (CPU cores * 1.5) + 1 = Unicorn workers. -For example a node with 4 cores would have 7 Unicorn workers. - -For all machines that have 2GB and up we recommend a minimum of three Unicorn workers. -If you have a 1GB machine we recommend to configure only two Unicorn workers to prevent excessive -swapping. - -As long as you have enough available CPU and memory capacity, it's okay to increase the number of -Unicorn workers and this will usually help to reduce the response time of the applications and -increase the ability to handle parallel requests. - -To change the Unicorn workers when you have the Omnibus package (which defaults to the -recommendation above) please see [the Unicorn settings in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/unicorn.html). + by Omnibus) databases (for example, [AWS Relational Database Service (RDS)](https://aws.amazon.com/rds/)) but we don't guarantee compatibility. +- You must also ensure the `postgres_fdw` extension is loaded into every + GitLab database. This extension + [can be enabled](https://www.postgresql.org/docs/11/sql-createextension.html) + using a PostgreSQL super user. ## Puma settings @@ -204,7 +164,7 @@ Omnibus GitLab defaults to the recommended Puma settings. Regardless of installa tune the Puma settings. If you're using Omnibus GitLab, see [Puma settings](https://docs.gitlab.com/omnibus/settings/puma.html) -for instructions on changing the Puma settings. +for instructions on changing the Puma settings. If you're using the GitLab Helm chart, see the [Webservice chart](https://docs.gitlab.com/charts/charts/gitlab/webservice/index.html). ### Puma workers @@ -228,10 +188,26 @@ of [legacy Rugged code](../development/gitaly.md#legacy-rugged-code). - If the operating system has a maximum 2 GB of memory, the recommended number of threads is `1`. A higher value will result in excess swapping, and decrease performance. - If legacy Rugged code is in use, the recommended number of threads is `1`. -- In all other cases, the recommended number of threads is `4`. We do not recommend setting this +- In all other cases, the recommended number of threads is `4`. We don't recommend setting this higher, due to how [Ruby MRI multi-threading](https://en.wikipedia.org/wiki/Global_interpreter_lock) works. +## Unicorn Workers + +For most instances we recommend using: (CPU cores * 1.5) + 1 = Unicorn workers. +For example a node with 4 cores would have 7 Unicorn workers. + +For all machines that have 2GB and up we recommend a minimum of three Unicorn workers. +If you have a 1GB machine we recommend to configure only two Unicorn workers to prevent excessive +swapping. + +As long as you have enough available CPU and memory capacity, it's okay to increase the number of +Unicorn workers and this will usually help to reduce the response time of the applications and +increase the ability to handle parallel requests. + +To change the Unicorn workers when you have the Omnibus package (which defaults to the +recommendation above) please see [the Unicorn settings in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/unicorn.html). + ## Redis and Sidekiq Redis stores all user sessions and the background task queue. @@ -257,11 +233,11 @@ to install GitLab on. Depending on how you decide to configure GitLab Runner and what tools you use to exercise your application in the CI environment, GitLab Runner can consume significant amount of available memory. -Memory consumption calculations, that are available above, will not be valid if +Memory consumption calculations, that are available above, won't be valid if you decide to run GitLab Runner and the GitLab Rails application on the same machine. -It is also not safe to install everything on a single machine, because of the +It's also not safe to install everything on a single machine, because of the [security reasons](https://docs.gitlab.com/runner/security/), especially when you plan to use shell executor with GitLab Runner. @@ -282,24 +258,24 @@ For reference, GitLab.com's [auto-scaling shared runner](../user/gitlab_com/inde ## Supported web browsers -CAUTION: **Caution:** With GitLab 13.0 (May 2020) we are removing official support for Internet Explorer 11. +CAUTION: **Caution:** With GitLab 13.0 (May 2020) we have removed official support for Internet Explorer 11. With the release of GitLab 13.4 (September 2020) we will remove all code that supports Internet Explorer 11. -You can provide feedback [on this issue](https://gitlab.com/gitlab-org/gitlab/issues/197987) or via your usual support channels. +You can provide feedback [on this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/197987) or via your usual support channels. GitLab supports the following web browsers: -- Firefox -- Chrome/Chromium -- Safari -- Microsoft Edge -- Internet Explorer 11 (until May 2020) +- [Mozilla Firefox](https://www.mozilla.org/en-US/firefox/new/) +- [Google Chrome](https://www.google.com/chrome/) +- [Chromium](https://www.chromium.org/getting-involved/dev-channel) +- [Apple Safari](https://www.apple.com/safari/) +- [Microsoft Edge](https://www.microsoft.com/en-us/edge) For the listed web browsers, GitLab supports: - The current and previous major versions of browsers except Internet Explorer. - The current minor version of a supported major version. -NOTE: **Note:** We do not support running GitLab with JavaScript disabled in the browser and have no plans of supporting that +NOTE: **Note:** We don't support running GitLab with JavaScript disabled in the browser and have no plans of supporting that in the future because we have features such as Issue Boards which require JavaScript extensively. <!-- ## Troubleshooting diff --git a/doc/integration/akismet.md b/doc/integration/akismet.md index 5b697d387e9..7cb8f8b70ce 100644 --- a/doc/integration/akismet.md +++ b/doc/integration/akismet.md @@ -1,42 +1,36 @@ # Akismet -> *Note:* Before 8.11 only issues submitted via the API and for non-project -members were submitted to Akismet. - GitLab leverages [Akismet](https://akismet.com/) to protect against spam. Currently GitLab uses Akismet to prevent the creation of spam issues on public projects. Issues -created via the WebUI or the API can be submitted to Akismet for review. +created via the web UI or the API can be submitted to Akismet for review. Detected spam will be rejected, and an entry in the "Spam Log" section in the Admin page will be created. -Privacy note: GitLab submits the user's IP and user agent to Akismet. Note that -adding a user to a project will disable the Akismet check and prevent this -from happening. +Privacy note: GitLab submits the user's IP and user agent to Akismet. + +NOTE: **Note:** +In GitLab 8.11 and later, all issues are submitted to Akismet. +In earlier GitLab versions, this only applied to API and non-project members. ## Configuration To use Akismet: 1. Go to the URL: <https://akismet.com/account/> - 1. Sign-in or create a new account. - 1. Click on **Show** to reveal the API key. - 1. Go to **Admin Area > Settings > Reporting** (`/admin/application_settings/reporting`). - 1. Check the **Enable Akismet** checkbox. - 1. Fill in the API key from step 3. - 1. Save the configuration.  ## Training -> *Note:* Training the Akismet filter is only available in 8.11 and above. +NOTE: **Note:** +Training the Akismet filter is only available in GitLab 8.11 and later. As a way to better recognize between spam and ham, you can train the Akismet filter whenever there is a false positive or false negative. diff --git a/doc/integration/azure.md b/doc/integration/azure.md index 809a3b703fc..2059707e38c 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -2,33 +2,18 @@ To enable the Microsoft Azure OAuth2 OmniAuth provider you must register your application with Azure. Azure will generate a client ID and secret key for you to use. -1. Sign in to the [Azure Management Portal](https://portal.azure.com). +Sign in to the [Azure Portal](https://portal.azure.com), and follow the instructions in +the [Microsoft Quickstart documentation](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app). -1. Select "Active Directory" on the left and choose the directory you want to use to register GitLab. +As you go through the Microsoft procedure, keep the following in mind: -1. Select "Applications" at the top bar and click the "Add" button the bottom. - -1. Select "Add an application my organization is developing". - -1. Provide the project information and click the "Next" button. - - Name: 'GitLab' works just fine here. - - Type: 'WEB APPLICATION AND/OR WEB API' - -1. On the "App properties" page enter the needed URI's and click the "Complete" button. - - SIGN-IN URL: Enter the URL of your GitLab installation (e.g `https://gitlab.mycompany.com/`) - - APP ID URI: Enter the endpoint URL for Microsoft to use, just has to be unique (e.g `https://mycompany.onmicrosoft.com/gitlab`) - -1. Select "Configure" in the top menu. - -1. Add a "Reply URL" pointing to the Azure OAuth callback of your GitLab installation (e.g. `https://gitlab.mycompany.com/users/auth/azure_oauth2/callback`). - -1. Create a "Client secret" by selecting a duration, the secret will be generated as soon as you click the "Save" button in the bottom menu. - -1. Note the "CLIENT ID" and the "CLIENT SECRET". - -1. Select "View endpoints" from the bottom menu. - -1. You will see lots of endpoint URLs in the form `https://login.microsoftonline.com/TENANT ID/...`, note down the TENANT ID part of one of those endpoints. +- If you have multiple instances of Azure Active Directory, you can switch to the desired tenant. +- You're setting up a Web application. +- For the redirect URI, you'll need the URL of the Azure OAuth callback of your GitLab installation (for example, `https://gitlab.mycompany.com/users/auth/azure_oauth2/callback`). The type dropdown should be set to "Web". +- The `client ID` and `client secret` are terms associated with OAuth 2. In some Microsoft documentation, + the terms may be listed as `Application ID` and `Application Secret`. +- If you need to generate a new client secret, follow the Microsoft documentation on how to [Create a new application secret](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#create-a-new-application-secret). +- Save the client ID and client secret for your new app. Once you leave the Azure portal, you won't be able to find the client secret again. 1. On your GitLab server, open the configuration file. @@ -84,4 +69,7 @@ To enable the Microsoft Azure OAuth2 OmniAuth provider you must register your ap 1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. -On the sign in page there should now be a Microsoft icon below the regular sign in form. Click the icon to begin the authentication process. Microsoft will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. +On the sign-in page, you should now see a Microsoft icon below the regular sign in form. Click the icon +to begin the authentication process. Microsoft then asks you to sign in and authorize the GitLab application. If everything goes well, you are returned to GitLab and signed in. +See [Enable OmniAuth for an Existing User](omniauth.md#enable-omniauth-for-an-existing-user) +for information on how existing GitLab users can connect to their newly-available Azure AD accounts. diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index ae9d5bd8f60..07ef5dbb99d 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -54,7 +54,7 @@ The way you install the Go indexer depends on your version of GitLab: ### Omnibus GitLab Since GitLab 11.8 the Go indexer is included in Omnibus GitLab. -The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/issues/6481). +The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481). ### From source @@ -142,6 +142,7 @@ The following Elasticsearch settings are available: | Parameter | Description | | ----------------------------------------------------- | ----------- | | `Elasticsearch indexing` | Enables/disables Elasticsearch indexing. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables background indexer which tracks data changes. So by enabling this you will not get your existing data indexed, use special Rake task for that as explained in [Adding GitLab's data to the Elasticsearch index](#adding-gitlabs-data-to-the-elasticsearch-index). | +| `Elasticsearch pause indexing` | Enables/disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until unpaused. | | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. | | `URL` | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., `http://host1, https://host2:9200`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://<username>:<password>@<elastic_host>:9200/`). | | `Number of Elasticsearch shards` | Elasticsearch indexes are split into multiple shards for performance reasons. In general, larger indexes need to have more shards. Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#create-index-settings) | @@ -418,14 +419,14 @@ The following are some available Rake tasks: | [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects and queues Sidekiq jobs to index them in the background. | | [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | | [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. | -| [`sudo gitlab-rake gitlab:elastic:create_empty_index[<INDEX_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates an empty index on the Elasticsearch side only if it doesn't already exist. | -| [`sudo gitlab-rake gitlab:elastic:delete_index[<INDEX_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab index on the Elasticsearch instance. | -| [`sudo gitlab-rake gitlab:elastic:recreate_index[<INDEX_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index[<INDEX_NAME>]` and `gitlab:elastic:create_empty_index[<INDEX_NAME>]`. | +| [`sudo gitlab-rake gitlab:elastic:create_empty_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates an empty index and assigns an alias for it on the Elasticsearch side only if it doesn't already exist. | +| [`sudo gitlab-rake gitlab:elastic:delete_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab index and alias (if exists) on the Elasticsearch instance. | +| [`sudo gitlab-rake gitlab:elastic:recreate_index[<TARGET_NAME>]`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index[<TARGET_NAME>]` and `gitlab:elastic:create_empty_index[<TARGET_NAME>]`. | | [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. | | [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. | NOTE: **Note:** -The `INDEX_NAME` parameter is optional and will use the default index name from the current `RAILS_ENV` if not set. +The `TARGET_NAME` parameter is optional and will use the default index/alias name from the current `RAILS_ENV` if not set. ### Environment variables @@ -511,7 +512,9 @@ Here are some common pitfalls and how to overcome them: pp s.search_objects.class.name ``` - If you see `Elasticsearch::Model::Response::Records`, you are using Elasticsearch. + If you see `"ActiveRecord::Relation"`, you are **not** using Elasticsearch. + + If you see `"Kaminari::PaginatableArray"` you are using Elasticsearch. NOTE: **Note**: The above instructions are used to verify that GitLab is using Elasticsearch only when indexing all namespaces. This is not to be used for scenarios that only index a [subset of namespaces](#limiting-namespaces-and-projects). @@ -627,13 +630,13 @@ Here are some common pitfalls and how to overcome them: You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). Once you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](#gitlab-elasticsearch-rake-tasks)) and [reindex the content of your instance](#adding-gitlabs-data-to-the-elasticsearch-index). -### Low level troubleshooting +### Low-level troubleshooting -There is more [low level troubleshooting documentation](../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. +There is a [more structured, lower-level troubleshooting document](../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. ### Known Issues -- **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/gitlab-org/gitlab/issues/10693)** +- **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/gitlab-org/gitlab/-/issues/10693)** The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have noticed [several edge cases](https://gitlab.com/gitlab-org/gitlab/-/issues/10693#note_158382332) that are not returning expected search results due to our pattern and filter configuration. diff --git a/doc/integration/gmail_action_buttons_for_gitlab.md b/doc/integration/gmail_action_buttons_for_gitlab.md index 1ab01494edc..526db8a7338 100644 --- a/doc/integration/gmail_action_buttons_for_gitlab.md +++ b/doc/integration/gmail_action_buttons_for_gitlab.md @@ -21,4 +21,4 @@ In particular, note: - Emails must be authenticated via DKIM or SPF. - Before sending the final form ("Gmail Schema Whitelist Request"), you must send a real email from your production server. This means that you will have to find a way to send this email from the email address you are registering. You can do this by, for example, forwarding the real email from the email address you are registering or going into the rails console on the GitLab server and triggering the email sending from there. -You can check how it looks going through all the steps laid out in the "Registering with Google" doc in [this GitLab.com issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/1517). +You can check how it looks going through all the steps laid out in the "Registering with Google" doc in [this GitLab.com issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/1517). diff --git a/doc/integration/img/ultra_auth_credentials.png b/doc/integration/img/ultra_auth_credentials.png Binary files differdeleted file mode 100644 index cff98a4b056..00000000000 --- a/doc/integration/img/ultra_auth_credentials.png +++ /dev/null diff --git a/doc/integration/img/ultra_auth_edit_callback_url.png b/doc/integration/img/ultra_auth_edit_callback_url.png Binary files differdeleted file mode 100644 index b7548122c5e..00000000000 --- a/doc/integration/img/ultra_auth_edit_callback_url.png +++ /dev/null diff --git a/doc/integration/img/ultra_auth_edit_callback_url_highlighted.png b/doc/integration/img/ultra_auth_edit_callback_url_highlighted.png Binary files differdeleted file mode 100644 index 4abf224756c..00000000000 --- a/doc/integration/img/ultra_auth_edit_callback_url_highlighted.png +++ /dev/null diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index e9b1c9676bd..f032345b9e0 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -1,6 +1,6 @@ # GitLab Jira development panel integration **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2381) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2381) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0. Complementary to our [existing Jira](../user/project/integrations/jira.md) project integration, you're now able to integrate GitLab projects with [Jira Development Panel](https://confluence.atlassian.com/adminjiraserver070/). Both can be used @@ -173,8 +173,8 @@ Click the links to see your GitLab repository data. ### 11.10 -- [Instance admins can now setup integration for all namespaces](https://gitlab.com/gitlab-org/gitlab/issues/8902) +- [Instance admins can now setup integration for all namespaces](https://gitlab.com/gitlab-org/gitlab/-/issues/8902) ### 11.1 -- [Support GitLab subgroups in Jira development panel](https://gitlab.com/gitlab-org/gitlab/issues/3561) +- [Support GitLab subgroups in Jira development panel](https://gitlab.com/gitlab-org/gitlab/-/issues/3561) diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index 01bd2a8c0a0..56f7b50496c 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -98,12 +98,12 @@ successful. ## Linking Kerberos and LDAP accounts together -If your users log in with Kerberos, but you also have [LDAP integration](../administration/auth/ldap.md) +If your users log in with Kerberos, but you also have [LDAP integration](../administration/auth/ldap/index.md) enabled, then your users will be automatically linked to their LDAP accounts on first login. For this to work, some prerequisites must be met: The Kerberos username must match the LDAP user's UID. You can choose which LDAP -attribute is used as the UID in GitLab's [LDAP configuration](../administration/auth/ldap.md#configuration) +attribute is used as the UID in GitLab's [LDAP configuration](../administration/auth/ldap/index.md#configuration-core-only) but for Active Directory, this should be `sAMAccountName`. The Kerberos realm must match the domain part of the LDAP user's Distinguished diff --git a/doc/integration/ldap.md b/doc/integration/ldap.md index 76c124d2ce9..25e4cc52375 100644 --- a/doc/integration/ldap.md +++ b/doc/integration/ldap.md @@ -1,5 +1,5 @@ --- -redirect_to: '../administration/auth/ldap.md' +redirect_to: '../administration/auth/ldap/index.md' --- -This document was moved to [`administration/auth/ldap`](../administration/auth/ldap.md). +This document was moved to [`administration/auth/ldap`](../administration/auth/ldap/index.md). diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 2afdeccb764..2ace05a8320 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -34,7 +34,6 @@ contains some settings that are common for all providers. - [OAuth2Generic](oauth2_generic.md) - [JWT](../administration/auth/jwt.md) - [OpenID Connect](../administration/auth/oidc.md) -- [UltraAuth](ultra_auth.md) - [Salesforce](salesforce.md) - [AWS Cognito](../administration/auth/cognito.md) diff --git a/doc/integration/saml.md b/doc/integration/saml.md index 5cb57baf353..97384d8f6cf 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -1,4 +1,4 @@ -# SAML OmniAuth Provider +# SAML OmniAuth Provider **(CORE ONLY)** Note that: diff --git a/doc/integration/ultra_auth.md b/doc/integration/ultra_auth.md deleted file mode 100644 index 95f2c0feb3b..00000000000 --- a/doc/integration/ultra_auth.md +++ /dev/null @@ -1,87 +0,0 @@ -# UltraAuth OmniAuth Provider - -You can integrate your GitLab instance with [UltraAuth](https://github.com/ultraauth) to enable users to perform secure biometric authentication to your GitLab instance with your UltraAuth account. Users have to perform the biometric authentication using their mobile device with fingerprint sensor. - -## Create UltraAuth Application - -To enable UltraAuth OmniAuth provider, you must use UltraAuth's credentials for your GitLab instance. -To get the credentials (a pair of Client ID and Client Secret), you must register an application on UltraAuth. - -1. Sign in to [UltraAuth](https://app.ultraauth.com). -1. Navigate to **Create an App** and click on **Ruby on Rails**. -1. Scroll down the page that is displayed to locate the **Client ID** and **Client Secret**. - Keep this page open as you continue configuration. - -  - -1. Click on "Edit Callback URL" link. - -  - -1. The callback URL will be `http(s)://<your_domain>/users/auth/ultraauth/callback` - -  - -1. Select **Register application**. -1. On your GitLab server, open the configuration file. - - For Omnibus package: - - ```shell - sudo editor /etc/gitlab/gitlab.rb - ``` - - For installations from source: - - ```shell - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` - -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: - - For Omnibus package: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "ultraauth", - "app_id" => "OPENID_CLIENT_ID", - "app_secret" => "OPENID_CLIENT_SECRET", - "args" => { - "client_options" => { - "redirect_uri" => "https://example.com/users/auth/ultraauth/callback" - } - } - } - ] - ``` - - For installation from source: - - ```yaml - - { name: 'ultraauth', - app_id: 'OPENID_CLIENT_ID', - app_secret: 'OPENID_CLIENT_SECRET', - args: { - client_options: { - redirect_uri: 'https://example.com/users/auth/ultraauth/callback' - } - } - } - ``` - - __Replace `https://example.com/users/auth/ultraauth/callback` with your application's Callback URL.__ - -1. Change `OPENID_CLIENT_ID` to the Client ID from the UltraAuth application page. -1. Change `OPENID_CLIENT_SECRET` to the Client Secret from the UltraAuth application page. -1. Save the configuration file. -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. - -On the sign in page, there should now be an UltraAuth icon below the regular sign in form. -Click the icon to begin the authentication process. UltraAuth will ask the user to sign in and authorize the GitLab application. -If everything goes well, the user will be returned to GitLab and will be signed in. - -GitLab requires the email address of each new user. Once the user is logged in using UltraAuth, GitLab will redirect the user to the profile page where they will have to provide the email and verify the email. Password authentication will be disabled for UltraAuth users and two-factor authentication (2FA) will be enforced. diff --git a/doc/integration/vault.md b/doc/integration/vault.md index 0a6ced42925..cead8f7592a 100644 --- a/doc/integration/vault.md +++ b/doc/integration/vault.md @@ -7,7 +7,7 @@ type: reference, howto # Vault Authentication with GitLab OpenID Connect -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/22323) in GitLab 9.0 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22323) in GitLab 9.0 [Vault](https://www.vaultproject.io/) is a secrets management application offered by HashiCorp. It allows you to store and manage sensitive information such as secret environment variables, encryption keys, and authentication tokens. @@ -21,7 +21,7 @@ The following assumes you already have Vault installed and running. First you'll need to create a GitLab application to obtain an application ID and secret for authenticating into Vault. To do this, sign in to GitLab and follow these steps: 1. On GitLab, click your avatar on the top-right corner, and select your user **Settings > Applications**. - 1. Fill out the application **Name** and [**Redirect URI**](https://www.vaultproject.io/docs/auth/jwt/#redirect-uris), + 1. Fill out the application **Name** and [**Redirect URI**](https://www.vaultproject.io/docs/auth/jwt#redirect-uris), making sure to select the **OpenID** scope. 1. Save application. 1. Copy client ID and secret, or keep the page open for reference. @@ -69,7 +69,7 @@ The following assumes you already have Vault installed and running. 1. **Write the OIDC Role Config:** - Now that Vault has a GitLab application ID and secret, it needs to know the [**Redirect URIs**](https://www.vaultproject.io/docs/auth/jwt/#redirect-uris) and scopes given to GitLab during the application creation process. The redirect URIs need to match where your Vault instance is running. The `oidc_scopes` field needs to include the `openid`. Similarly to the previous step, replace `your_application_id` with the generated application ID from GitLab: + Now that Vault has a GitLab application ID and secret, it needs to know the [**Redirect URIs**](https://www.vaultproject.io/docs/auth/jwt#redirect-uris) and scopes given to GitLab during the application creation process. The redirect URIs need to match where your Vault instance is running. The `oidc_scopes` field needs to include the `openid`. Similarly to the previous step, replace `your_application_id` with the generated application ID from GitLab: This configuration is saved under the name of the role you are creating. In this case, we are creating a `demo` role. Later, we'll show how you can access this role through the Vault CLI. @@ -110,7 +110,7 @@ The following assumes you already have Vault installed and running. 1. In the **Write the OIDC Role Config** (step 4), we created a role called `demo`. We set `role=demo` so Vault knows which configuration we'd like to login in with. 1. To set Vault to use the `OIDC` sign-in method, we set `-method=oidc`. - 1. To set the port that GitLab should redirect to, we set `port=8250` or another port number that matches the port given to GitLab when listing [Redirect URIs](https://www.vaultproject.io/docs/auth/jwt/#redirect-uris). + 1. To set the port that GitLab should redirect to, we set `port=8250` or another port number that matches the port given to GitLab when listing [Redirect URIs](https://www.vaultproject.io/docs/auth/jwt#redirect-uris). Once you run the command above, it will present a link in the terminal. Click the link in the terminal and a tab will open in the browser confirming you're signed into Vault via OIDC: diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index cf2a7a375cc..bba16e491d0 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -5,23 +5,23 @@ type: concepts # GitLab Release and Maintenance Policy GitLab has strict policies governing version naming, as well as release pace for major, minor, -patch and security releases. New releases are usually announced on the [GitLab blog](https://about.gitlab.com/releases/categories/releases/). +patch, and security releases. New releases are usually announced on the [GitLab blog](https://about.gitlab.com/releases/categories/releases/). Our current policy is: -- Backporting bug fixes for **only the current stable release** at any given time, see [patch releases](#patch-releases). -- Backporting to **to the previous two monthly releases in addition to the current stable release**, see [security releases](#security-releases). +- Backporting bug fixes for **only the current stable release** at any given time. (See [patch releases](#patch-releases).) +- Backporting **to the previous two monthly releases in addition to the current stable release**. (See [security releases](#security-releases).) ## Versioning GitLab uses [Semantic Versioning](https://semver.org/) for its releases: `(Major).(Minor).(Patch)`. -For example, for GitLab version 10.5.7: +For example, for GitLab version 12.10.6: -- `10` represents the major version. The major release was 10.0.0, but often referred to as 10.0. -- `5` represents the minor version. The minor release was 10.5.0, but often referred to as 10.5. -- `7` represents the patch number. +- `12` represents the major version. The major release was 12.0.0, but often referred to as 12.0. +- `10` represents the minor version. The minor release was 12.10.0, but often referred to as 12.10. +- `6` represents the patch number. Any part of the version number can increment into multiple digits, for example, 13.10.11. @@ -29,10 +29,118 @@ The following table describes the version types and their release cadence: | Version type | Description | Cadence | |:-------------|:------------|:--------| -| Major | For significant changes, or when any backward-incompatible changes are introduced to the public API. | Yearly. The next major release is GitLab 13.0 on May 22, 2020. Subsequent major releases will be scheduled for May 22 each year, by default. | +| Major | For significant changes, or when any backward-incompatible changes are introduced to the public API. | Yearly. The next major release is GitLab 14.0 on May 22, 2021. Subsequent major releases will be scheduled for May 22 each year, by default. | | Minor | For when new backward-compatible functionality is introduced to the public API, a minor feature is introduced, or when a set of smaller features is rolled out. | Monthly on the 22nd. | | Patch | For backward-compatible bug fixes that fix incorrect behavior. See [Patch releases](#patch-releases). | As needed. | +## Upgrade recommendations + +We encourage everyone to run the [latest stable release](https://about.gitlab.com/releases/categories/releases/) +to ensure that you can easily upgrade to the most secure and feature-rich GitLab experience. +In order to make sure you can easily run the most recent stable release, we are working +hard to keep the update process simple and reliable. + +If you are unable to follow our monthly release cycle, there are a couple of +cases you need to consider. + +It is considered safe to jump between patch versions and minor versions within +one major version. For example, it is safe to: + +- Upgrade the *minor* version. For example: + + - `12.7.5` -> `12.10.5` + - `11.3.4` -> `11.11.1` + - `10.6.6` -> `10.8.3` + - `11.3.4` -> `11.11.8` + - `10.6.6` -> `10.8.7` + - `9.2.3` -> `9.5.5` + - `8.9.4` -> `8.12.3` + +- Upgrade the *patch* version. For example: + + - `12.0.4` -> `12.0.12` + - `11.11.1` -> `11.11.8` + - `10.6.3` -> `10.6.6` + - `11.11.1` -> `11.11.8` + - `10.6.3` -> `10.6.6` + - `9.5.5` -> `9.5.9` + - `8.9.2` -> `8.9.6` + +NOTE **Note** Version specific changes in Omnibus GitLab Linux packages can be found in [the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/update/README.html#version-specific-changes). + +NOTE: **Note:** +Instructions are available for downloading an Omnibus GitLab Linux package locally and [manually installing](https://docs.gitlab.com/omnibus/manual_install.html) it. + +### Upgrading major versions + +Upgrading the *major* version requires more attention. +Backward-incompatible changes and migrations are reserved for major versions. +We cannot guarantee that upgrading between major versions will be seamless. +We suggest upgrading to the latest available *minor* version within +your major version before proceeding to the next major version. +Doing this will address any backward-incompatible changes or deprecations +to help ensure a successful upgrade to next major release. + +It's also important to ensure that any background migrations have been fully completed +before upgrading to a new major version. To see the current size of the `background_migration` queue, +[Check for background migrations before upgrading](../update/README.md#checking-for-background-migrations-before-upgrading). + +If your GitLab instance has any GitLab Runners associated with it, it is very +important to upgrade the GitLab Runners to match the GitLab minor version that was +upgraded to. This is to ensure [compatibility with GitLab versions](https://docs.gitlab.com/runner/#compatibility-with-gitlab-versions). + +### Version 12 onward: Extra step for major upgrades + +From version 12 onward, an additional step is required. More significant migrations +may occur during major release upgrades. + +To ensure these are successful: + +1. Increment to the first minor version (`x.0.x`) during the major version jump. +1. Proceed with upgrading to a newer release. + +**For example: `11.5.x` -> `11.11.x` -> `12.0.x` -> `12.10.x` -> `13.0.x`** + +### Example upgrade paths + +Please see the table below for some examples: + +| Target version | Your version | Recommended upgrade path | Note | +| --------------------- | ------------ | ------------------------ | ---- | +| `13.2.0` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.10.6` -> `13.0.0` -> `13.2.0` | Four intermediate versions are required: the final 11.11, 12.0, and 12.10 releases, plus 13.0. | +| `13.0.1` | `11.10.8` | `11.10.5` -> `11.11.8` -> `12.0.12` -> `12.10.6` -> `13.0.1` | Three intermediate versions are required: `11.11`, `12.0`, and `12.10`. | +| `12.10.6` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.10.6` | Two intermediate versions are required: `11.11` and `12.0` | +| `12.9.5.` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.9.5` | Three intermediate versions are required: `10.8`, `11.11`, and `12.0`, then `12.9.5` | +| `12.2.5` | `9.2.6` | `9.2.6` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.2.5` | Four intermediate versions are required: `9.5`, `10.8`, `11.11`, `12.0`, then `12.2`. | +| `11.3.4` | `8.13.4` | `8.13.4` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.3.4` | `8.17.7` is the last version in version 8, `9.5.10` is the last version in version 9, `10.8.7` is the last version in version 10. | + +### Upgrades from versions earlier than 8.12 + +- `8.11.x` and earlier: you might have to upgrade to `8.12.0` specifically before you can + upgrade to `8.17.7`. This was [reported in an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/207259). +- [CI changes prior to version 8.0](https://docs.gitlab.com/omnibus/update/README.html#updating-gitlab-ci-from-prior-540-to-version-714-via-omnibus-gitlab) + when it was merged into GitLab. + +### Multi-step upgrade paths with GitLab all-in-one Linux package repository + +Linux package managers default to installing the latest available version of a package for installation and upgrades. +Upgrading directly to the latest major version can be problematic for older GitLab versions that require a multi-stage upgrade path. + +When following an upgrade path spanning multiple versions, for each upgrade, specify the intended GitLab version number in your package manager's install or upgrade command. + +Examples: + +```shell +# apt-get (Ubuntu/Debian) +sudo apt-get upgrade gitlab-ee=12.0.12-ee.0 +# yum (RHEL/CentOS 6 and 7) +yum install gitlab-ee-12.0.12-ee.0.el7 +# dnf (RHEL/CentOS 8) +dnf install gitlab-ee-12.0.12-ee.0.el8 +# zypper (SUSE) +zypper install gitlab-ee=12.0.12-ee.0 +``` + ## Patch releases Patch releases **only include bug fixes** for the current stable released version of @@ -62,7 +170,7 @@ have to adhere to various internal requirements (for example, org. compliance, v In cases where a strategic user has a requirement to test a feature before it is officially released, we can offer to create a Release Candidate (RC) version that will include the specific feature. This should be needed only in extreme cases, and can be requested for -consideration by raising an issue in the [release/tasks](https://gitlab.com/gitlab-org/release/tasks/issues/new?issuable_template=Backporting-request) issue tracker. +consideration by raising an issue in the [release/tasks](https://gitlab.com/gitlab-org/release/tasks/-/issues/new?issuable_template=Backporting-request) issue tracker. It is important to note that the Release Candidate will also contain other features and changes as it is not possible to easily isolate a specific feature (similar reasons as noted above). The Release Candidate will be no different than any code that is deployed to GitLab.com or is publicly @@ -95,7 +203,7 @@ For instance, if we release `11.2.1` with a fix for a severe bug introduced in `11.0.0`, we could backport the fix to a new `11.0.x`, and `11.1.x` patch release. To request backporting to more than one stable release for consideration, raise an issue in the -[release/tasks](https://gitlab.com/gitlab-org/release/tasks/issues/new?issuable_template=Backporting-request) issue tracker. +[release/tasks](https://gitlab.com/gitlab-org/release/tasks/-/issues/new?issuable_template=Backporting-request) issue tracker. ### Security releases @@ -107,86 +215,12 @@ For very serious security issues, there is to backport security fixes to even more monthly releases of GitLab. This decision is made on a case-by-case basis. -## Upgrade recommendations - -We encourage everyone to run the [latest stable release](https://about.gitlab.com/releases/categories/releases/) to ensure that you can -easily upgrade to the most secure and feature-rich GitLab experience. In order -to make sure you can easily run the most recent stable release, we are working -hard to keep the update process simple and reliable. - -If you are unable to follow our monthly release cycle, there are a couple of -cases you need to consider. - -It is considered safe to jump between patch versions and minor versions within -one major version. For example, it is safe to: - -- Upgrade the patch version: - - `8.9.0` -> `8.9.7` - - `8.9.0` -> `8.9.1` - - `8.9.2` -> `8.9.6` - - `9.5.5` -> `9.5.9` - - `10.6.3` -> `10.6.6` - - `11.11.1` -> `11.11.8` - - `12.0.4` -> `12.0.12` -- Upgrade the minor version: - - `8.9.4` -> `8.12.3` - - `9.2.3` -> `9.5.5` - - `10.6.6` -> `10.8.7` - - `11.3.4` -> `11.11.8` - -Upgrading the major version requires more attention. -We cannot guarantee that upgrading between major versions will be seamless. As previously mentioned, major versions are reserved for backwards incompatible changes. -We recommend that you first upgrade to the latest available minor version within -your major version. By doing this, you can address any deprecation messages -that could change behavior in the next major release. - -It's also important to ensure that any background migrations have been fully completed -before upgrading to a new major version. To see the current size of the `background_migration` queue, -[Check for background migrations before upgrading](../update/README.md#checking-for-background-migrations-before-upgrading). - -### Version 12 onwards: Extra step for major upgrades - -From version 12 onwards, an additional step is required. More significant migrations -may occur during major release upgrades. - -To ensure these are successful: - -1. Increment to the first minor version (`x.0.x`) during the major version jump. -1. Proceed with upgrading to a newer release. - -For example: `11.11.x` -> `12.0.x` -> `12.8.x` - -### Upgrades from old versions - -- `8.11.x` and earlier: you might have to upgrade to `8.12.0` specifically before you can - upgrade to `8.17.7`. This was [reported in an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/207259). -- [CI changes prior to version 8.0](https://docs.gitlab.com/omnibus/update/README.html#updating-gitlab-ci-from-prior-540-to-version-714-via-omnibus-gitlab) - when it was merged into GitLab. -- Version specific changes in - [the Omnibus documentation](https://docs.gitlab.com/omnibus/update/README.html#version-specific-changes). - -### Example upgrade paths - -Please see the table below for some examples: - -| Latest stable version | Your version | Recommended upgrade path | Note | -| --------------------- | ------------ | ------------------------ | ---- | -| 9.4.5 | 8.13.4 | `8.13.4` -> `8.17.7` -> `9.4.5` | `8.17.7` is the last version in version `8` | -| 10.1.4 | 8.13.4 | `8.13.4 -> 8.17.7 -> 9.5.10 -> 10.1.4` | `8.17.7` is the last version in version `8`, `9.5.10` is the last version in version `9` | -| 11.3.4 | 8.13.4 | `8.13.4` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.3.4` | `8.17.7` is the last version in version `8`, `9.5.10` is the last version in version `9`, `10.8.7` is the last version in version `10` | -| 12.5.10 | 11.3.4 | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.5.10` | `11.11.8` is the last version in version `11`. `12.0.x` [is a required step](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23211#note_272842444). | -| 12.8.5 | 9.2.6 | `9.2.6` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.8.5` | Four intermediate versions are required: the final 9.5, 10.8, 11.11 releases, plus 12.0. | - -NOTE: **Note:** -Instructions for installing a specific version of GitLab or downloading the package locally for installation can be found at [GitLab Repositories](https://packages.gitlab.com/gitlab). - ## More information Check [our release posts](https://about.gitlab.com/releases/categories/releases/). Each month, we publish either a major or minor release of GitLab. At the end -of those release posts there are three sections to look for: deprecations, important notes, -and upgrade barometer. These will draw your attention to: +of those release posts there are three sections to look for: Deprecations, Removals, and Important notes on upgrading. These will include: - Steps you need to perform as part of an upgrade. For example [8.12](https://about.gitlab.com/releases/2016/09/22/gitlab-8-12-released/#upgrade-barometer) diff --git a/doc/public_access/public_access.md b/doc/public_access/public_access.md index 0ca2da1db63..3f93cd7f61f 100644 --- a/doc/public_access/public_access.md +++ b/doc/public_access/public_access.md @@ -33,7 +33,7 @@ NOTE: **Note:** From July 2019, the `Internal` visibility setting is disabled for new projects, groups, and snippets on GitLab.com. Existing projects, groups, and snippets using the `Internal` visibility setting keep this setting. You can read more about the change in the -[relevant issue](https://gitlab.com/gitlab-org/gitlab/issues/12388). +[relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/12388). ### Private projects @@ -59,6 +59,8 @@ it. The restriction for visibility levels on the application setting level also applies to groups, so if that's set to internal, the explore page will be empty for anonymous users. The group page now has a visibility level icon. +Admin users cannot create subgroups or projects with higher visibility level than that of the parent group. + ## Visibility of users The public page of a user, located at `/username`, is always visible whether @@ -89,7 +91,7 @@ For details, see [Restricted visibility levels](../user/admin_area/settings/visi ## Reducing visibility -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/33358) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33358) in GitLab 12.6. Reducing a project's visibility level will remove the fork relationship between the project and any forked project. This is a potentially destructive action which requires confirmation before diff --git a/doc/push_rules/push_rules.md b/doc/push_rules/push_rules.md index 5685e848a33..e0ca914b67b 100644 --- a/doc/push_rules/push_rules.md +++ b/doc/push_rules/push_rules.md @@ -95,15 +95,15 @@ GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular ## Prevent pushing secrets to the repository -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/385) in [GitLab Starter](https://about.gitlab.com/pricing/) 8.12. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385) in [GitLab Starter](https://about.gitlab.com/pricing/) 8.12. Secrets such as credential files, SSH private keys, and other files containing secrets should never be committed to source control. -GitLab allows you to turn on a predefined blacklist of files which won't be allowed to be +GitLab allows you to turn on a predefined denylist of files which won't be allowed to be pushed to a repository, stopping those commits from reaching the remote repository. By selecting the checkbox *Prevent committing secrets to Git*, GitLab prevents pushes to the repository when a file matches a regular expression as read from -[`files_blacklist.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/checks/files_blacklist.yml) (make sure you are at the right branch +[`files_denylist.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/checks/files_denylist.yml) (make sure you are at the right branch as your GitLab version when viewing this file). NOTE: **Note:** diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index 4bba4f2bc5a..1d8aad25d57 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -28,8 +28,9 @@ The following are available Rake tasks: | [Import repositories](import.md) | Import bare repositories into your GitLab instance. | | [Import large project exports](../development/import_project.md#importing-via-a-rake-task) | Import large GitLab [project exports](../user/project/settings/import_export.md). | | [Integrity checks](../administration/raketasks/check.md) | Check the integrity of repositories, files, and LDAP. | -| [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap.md)-related tasks. | +| [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap/index.md)-related tasks. | | [List repositories](list_repos.md) | List of all GitLab-managed Git repositories on disk. | +| [Migrate Snippets to Git](migrate_snippets.md) | Migrate GitLab Snippets to Git repositories and show migration status | | [Project import/export](../administration/raketasks/project_import_export.md) | Prepare for [project exports and imports](../user/project/settings/import_export.md). | | [Sample Prometheus data](generate_sample_prometheus_data.md) | Generate sample Prometheus data. | | [Repository storage](../administration/raketasks/storage.md) | List and migrate existing projects and attachments from legacy storage to hashed storage. | @@ -38,4 +39,3 @@ The following are available Rake tasks: | [User management](user_management.md) | Perform user management tasks. | | [Webhooks administration](web_hooks.md) | Maintain project Webhooks. | | [X.509 signatures](x509_signatures.md) | Update X.509 commit signatures, useful if certificate store has changed. | -| [Migrate Snippets to Git](migrate_snippets.md) | Migrate GitLab Snippets to Git repositories and show migration status | diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 3c81bab644e..e83f51dc84e 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -641,7 +641,7 @@ recommended that you configure the appropriate retention policy for your object storage (for example, [AWS S3](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-lifecycle.html)). You may want to set a limited lifetime for backups to prevent regular -backups using all your disk space. +backups using all your disk space. The next time the backup task is run, backups older than the `backup_keep_time` will be pruned. For Omnibus GitLab packages: @@ -803,7 +803,7 @@ restore: ```shell # This command will overwrite the contents of your GitLab database! -sudo gitlab-backup restore BACKUP=1493107454_2018_04_25_10.6.4-ce +sudo gitlab-backup restore BACKUP=11493107454_2018_04_25_10.6.4-ce ``` NOTE: **Note** @@ -811,7 +811,7 @@ For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:restore`. CAUTION: **Warning:** `gitlab-rake gitlab:backup:restore` does not set the right file system permissions on your Registry directory. -This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/62759). On GitLab 12.2 or newer, you can +This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759). On GitLab 12.2 or newer, you can use `gitlab-backup restore` to avoid this issue. Next, restore `/etc/gitlab/gitlab-secrets.json` if necessary as mentioned above. @@ -832,7 +832,7 @@ version of GitLab, the restore command will abort with an error. Install the For GitLab installations using the Docker image or the GitLab Helm chart on a Kubernetes cluster, the restore task expects the restore directories to be empty. -However, with docker and Kubernetes volume mounts, some system level directories +However, with Docker and Kubernetes volume mounts, some system level directories may be created at the volume roots, like `lost+found` directory found in Linux operating systems. These directories are usually owned by `root`, which can cause access permission errors since the restore Rake task runs as `git` user. @@ -842,7 +842,7 @@ directories are empty. For both these installation types, the backup tarball has to be available in the backup location (default location is `/var/opt/gitlab/backups`). -For docker installations, the restore task can be run from host: +For Docker installations, the restore task can be run from host: ```shell docker exec -it <name of container> gitlab-backup restore @@ -853,7 +853,7 @@ For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:restore`. CAUTION: **Warning:** `gitlab-rake gitlab:backup:restore` does not set the right file system permissions on your Registry directory. -This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/62759). On GitLab 12.2 or newer, you can +This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759). On GitLab 12.2 or newer, you can use `gitlab-backup restore` to avoid this issue. The GitLab Helm chart uses a different process, documented in @@ -1063,7 +1063,7 @@ err.message="unknown error" This is caused by the restore being run as the unprivileged user `git` which was unable to assign the correct ownership to the registry files during the restore -([issue 62759](https://gitlab.com/gitlab-org/gitlab-foss/issues/62759 "Incorrect permissions on registry filesystem after restore")). +([issue 62759](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759 "Incorrect permissions on registry filesystem after restore")). To get your registry working again: diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index 7908af8da84..5bdae998ec9 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -4,7 +4,7 @@ GitLab provides Rake tasks for cleaning up GitLab instances. ## Remove unreferenced LFS files from filesystem -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36628) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36628) in GitLab 12.10. DANGER: **Danger:** Do not run this within 12 hours of a GitLab upgrade. This is to ensure that all background migrations @@ -44,6 +44,8 @@ later (once a day). If you need to garbage collect them immediately, run ## Remove unreferenced LFS files +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36628) in GitLab 12.10. + Unreferenced LFS files are removed on a daily basis but you can remove them immediately if you need to. For example: @@ -64,6 +66,8 @@ I, [2020-01-08T20:51:17.148765 #43765] INFO -- : Removed unreferenced LFS files ## Remove garbage from filesystem +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20863) in GitLab 11.2. + Clean up local project upload files if they don't exist in GitLab database. The task attempts to fix the file if it can find its project, otherwise it moves the file to a lost and found directory. @@ -96,6 +100,10 @@ I, [2018-07-27T12:08:33.755624 #89817] INFO -- : Did fix /opt/gitlab/embedded/s I, [2018-07-27T12:08:33.760257 #89817] INFO -- : Did move to lost and found /opt/gitlab/embedded/service/gitlab-rails/public/uploads/foo/bar/1dd6f0f7eefd2acc4c2233f89a0f7b0b/image.png -> /opt/gitlab/embedded/service/gitlab-rails/public/uploads/-/project-lost-found/foo/bar/1dd6f0f7eefd2acc4c2233f89a0f7b0b/image.png ``` +## Remove garbage from object storage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20918) in GitLab 11.2. + Remove object store upload files if they don't exist in GitLab database. ```shell @@ -127,6 +135,9 @@ I, [2018-08-02T10:26:47.764356 #45087] INFO -- : Moved to lost and found: @hash ## Remove orphan artifact files +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29681) in GitLab 12.1. +> - [`ionice` support fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28023) in GitLab 12.10. + When you notice there are more job artifacts files on disk than there should be, you can run: @@ -172,6 +183,8 @@ level with `NICENESS`. Below are the valid levels, but consult ## Remove expired ActiveSession lookup keys +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30668) in GitLab 12.2. + ```shell # omnibus-gitlab sudo gitlab-rake gitlab:cleanup:sessions:active_sessions_lookup_keys diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index 5229ce2ab08..263f9e54a20 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -115,16 +115,16 @@ repository in GitLab 10.4 and later: - Ancestor renamed - Ancestor transferred to another namespace -Bare repositories are **not** importable by GitLab 10.4 and later when all the following are true about the repository: +Bare repositories are **not** importable by GitLab 10.4 to GitLab 11.6, if all the following are true about the repository: - It was created in GitLab 10.3 or earlier. -- It was not renamed, transferred, or migrated to hashed storage in GitLab 10.4 and later. -- Its ancestor namespaces were not renamed or transferred in GitLab 10.4 and later. +- It was not renamed, transferred, or migrated to [hashed storage](../administration/repository_storage_types.md#hashed-storage) in GitLab 10.4 to GitLab 11.6. +- Its ancestor namespaces were not renamed or transferred in GitLab 10.4 to GitLab 11.6. -There is an [open issue to add a migration to make all bare repositories -importable](https://gitlab.com/gitlab-org/gitlab-foss/issues/41776). +[Since GitLab 11.6](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41776), all +bare repositories are importable. -Until then, you may wish to manually migrate repositories yourself. You can use +To manually migrate repositories yourself (for GitLab 10.4 to GitLab 11.6), you can use the [Rails console](../administration/troubleshooting/debug.md#starting-a-rails-console-session) to do so. In a Rails console session, run the following to migrate a project: diff --git a/doc/raketasks/migrate_snippets.md b/doc/raketasks/migrate_snippets.md index fce91ab703e..476615d3926 100644 --- a/doc/raketasks/migrate_snippets.md +++ b/doc/raketasks/migrate_snippets.md @@ -35,7 +35,7 @@ bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 ``` There is a default limit (100) to the number of ids supported in the migration -process. You can modify this limit by using the env variable `LIMIT`. +process. You can modify this limit by using the environment variable `LIMIT`. ```shell sudo gitlab-rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 LIMIT=50 @@ -83,7 +83,7 @@ bundle exec rake gitlab:snippets:list_non_migrated RAILS_ENV=production As the number of non-migrated snippets can be large, we limit by default the size of the number of ids returned to 100. You can -modify this limit by using the env variable `LIMIT`. +modify this limit by using the environment variable `LIMIT`. ```shell sudo gitlab-rake gitlab:snippets:list_non_migrated LIMIT=200 diff --git a/doc/security/img/allowlist_v13_0.png b/doc/security/img/allowlist_v13_0.png Binary files differnew file mode 100644 index 00000000000..973b53a57a4 --- /dev/null +++ b/doc/security/img/allowlist_v13_0.png diff --git a/doc/security/img/whitelist.png b/doc/security/img/whitelist.png Binary files differdeleted file mode 100644 index 897000e804d..00000000000 --- a/doc/security/img/whitelist.png +++ /dev/null diff --git a/doc/security/rack_attack.md b/doc/security/rack_attack.md index 5d18746e4e0..605b669d498 100644 --- a/doc/security/rack_attack.md +++ b/doc/security/rack_attack.md @@ -121,9 +121,6 @@ The following settings can be configured: **Installations from source** -NOTE: **Note:** Rack Attack initializer was temporarily renamed to `rack_attack_new`, to -support backwards compatibility with the one [Omnibus initializer](https://docs.gitlab.com/omnibus/settings/configuration.html#setting-up-paths-to-be-protected-by-rack-attack). It'll be renamed back to `rack_attack.rb` once Omnibus throttle is removed. Please see the [GitLab issue](https://gitlab.com/gitlab-org/gitlab/issues/29952) for more information. - These settings can be found in `config/initializers/rack_attack.rb`. If you are missing `config/initializers/rack_attack.rb`, the following steps need to be taken in order to enable protection for your GitLab instance: diff --git a/doc/security/webhooks.md b/doc/security/webhooks.md index 27f79dbdf66..af9be499e80 100644 --- a/doc/security/webhooks.md +++ b/doc/security/webhooks.md @@ -52,29 +52,29 @@ NOTE: **Note:** set up by administrators. However, you can turn this off by disabling the **Allow requests to the local network from system hooks** option. -## Whitelist for local requests +## Allowlist for local requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/44496) in GitLab 12.2 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/44496) in GitLab 12.2 You can allow certain domains and IP addresses to be accessible to both *system hooks* and *webhooks* even when local requests are not allowed by adding them to the -whitelist. Navigate to **Admin Area > Settings > Network** (`/admin/application_settings/network`) +allowlist. Navigate to **Admin Area > Settings > Network** (`/admin/application_settings/network`) and expand **Outbound requests**: - + -The whitelist entries can be separated by semicolons, commas or whitespaces +The allowed entries can be separated by semicolons, commas or whitespaces (including newlines) and be in different formats like hostnames, IP addresses and/or -IP ranges. IPv6 is supported. Hostnames that contain unicode characters should +IP ranges. IPv6 is supported. Hostnames that contain Unicode characters should use IDNA encoding. -The whitelist can hold a maximum of 1000 entries. Each entry can be a maximum of +The allowlist can hold a maximum of 1000 entries. Each entry can be a maximum of 255 characters. -You can whitelist a particular port by specifying it in the whitelist entry. +You can allow a particular port by specifying it in the allowlist entry. For example `127.0.0.1:8080` will only allow connections to port 8080 on `127.0.0.1`. -If no port is mentioned, all ports on that IP/domain are whitelisted. An IP range -will whitelist all ports on all IPs in that range. +If no port is mentioned, all ports on that IP/domain are allowed. An IP range +will allow all ports on all IPs in that range. Example: diff --git a/doc/ssh/README.md b/doc/ssh/README.md index 06db8e59850..5db1bde6413 100644 --- a/doc/ssh/README.md +++ b/doc/ssh/README.md @@ -119,7 +119,7 @@ Enter file in which to save the key (/home/user/.ssh/id_rsa): For guidance, proceed to the [common steps](#common-steps-for-generating-an-ssh-key-pair). NOTE: **Note:** -If your have OpenSSH version 7.8 or below, consider the problems associated +If you have OpenSSH version 7.8 or below, consider the problems associated with [encoding](#rsa-keys-and-openssh-from-versions-65-to-78). ### Common steps for generating an SSH key pair @@ -141,7 +141,7 @@ You can assign the directory and file name of your choice. You can also dedicate that SSH key pair to a [specific host](#working-with-non-default-ssh-key-pair-paths). After assigning a file to save your SSH key, you'll get a chance to set up -a [passphrase](https://www.ssh.com/ssh/passphrase) for your SSH key: +a [passphrase](https://www.ssh.com/ssh/passphrase/) for your SSH key: ```plaintext Enter passphrase (empty for no passphrase): @@ -344,7 +344,7 @@ NOTE: **Note:** The example `Host` aliases are defined as `user_1.gitlab.com` and `user_2.gitlab.com` for efficiency and transparency. Advanced configurations are more difficult to maintain; using this type of alias makes it easier to -understand when using other tools such as `git remote` subcommands. SSH +understand when using other tools such as `git remote` sub-commands. SSH would understand any string as a `Host` alias thus `Tanuki1` and `Tanuki2`, despite giving very little context as to where they point, would also work. @@ -370,80 +370,7 @@ git remote set-url origin git@<user_1.gitlab.com>:gitlab-org/gitlab.git ## Deploy keys -Deploy keys allow read-only or read-write (if enabled) access to one or -multiple repositories with a single SSH key pair. - -This is useful for cloning repositories to your Continuous -Integration (CI) server. By using deploy keys, you don't have to set up a -dummy user account. - -If you don't have a key pair, you might want to use a -[deploy token](../user/project/deploy_tokens/index.md#deploy-tokens) instead. - -### Per-repository deploy keys - -Project maintainers and owners can add a deploy key for a repository. - -1. Navigate to the project's **Settings > Repository** page. -1. Expand the **Deploy Keys** section. -1. Specify a title for the new deploy key and paste a public SSH key. - -After this, the machine that uses the corresponding private SSH key has read-only or -read-write (if enabled) access to the project. - -You can't add the same deploy key twice using the form. -If you want to add the same key to another project, please enable it in the -list that says **Deploy keys from projects available to you**. All the deploy -keys of all the projects you have access to are available. This project -access can happen through being a direct member of the project, or through -a group. - -Deploy keys can be shared between projects, you just need to add them to each -project. - -### Global shared deploy keys - -Global Shared Deploy keys allow read-only or read-write access to -any repository in the entire GitLab installation. - -This is useful for integrating repositories to secured, shared Continuous -Integration (CI) services or other shared services. -GitLab administrators can set up the Global Shared Deploy key in GitLab and -add the private key to any shared systems. Individual repositories opt into -exposing their repository using these keys when a project maintainers (or higher) -authorizes a Global Shared Deploy key to be used with their project. - -Global Shared Keys can provide greater security compared to Per-Project Deploy -Keys since an administrator of the target integrated system is the only one -who needs to know and configure the private key. - -GitLab administrators set up Global Deploy keys in the Admin Area under the -section **Deploy Keys**. Ensure keys have a meaningful title as that will be -the primary way for project maintainers and owners to identify the correct Global -Deploy key to add. For instance, if the key gives access to a SaaS CI instance, -use the name of that service in the key name if that is all it is used for. -When creating Global Shared Deploy keys, give some thought to the granularity -of keys - they could be of very narrow usage such as just a specific service or -of broader usage for something like "Anywhere you need to give read access to -your repository". - -Once a GitLab administrator adds the Global Deployment key, project maintainers -and owners can add it by: - -1. Navigate to the project's **Settings > Repository** page. -1. Expanding the **Deploy Keys** section. -1. Clicking **Enable** next to the appropriate key listed under - **Public deploy keys available to any project**. - -NOTE: **Note:** -The heading **Public deploy keys available to any project** only appears -if there is at least one Global Deploy Key configured. - -CAUTION: **Warning:** -Defining Global Deploy Keys does not expose any given repository via -the key until that repository adds the Global Deploy Key to their project. -In this way the Global Deploy Keys enable access by other systems, but do -not implicitly give any access just by setting them up. +Read the [documentation on Deploy Keys](../user/project/deploy_keys/index.md). ## Applications @@ -458,7 +385,7 @@ GitLab integrates with the system-installed SSH daemon, designating a user connecting to the GitLab server over SSH are identified by their SSH key instead of their username. -SSH *client* operations performed on the GitLab server wil be executed as this +SSH *client* operations performed on the GitLab server will be executed as this user. Although it is possible to modify the SSH configuration for this user to, e.g., provide a private SSH key to authenticate these requests by, this practice is **not supported** and is strongly discouraged as it presents significant @@ -485,7 +412,7 @@ are *explicitly not supported* and may stop working at any time. ### Options for Microsoft Windows -If you're running Windows 10, the [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and its latest [WSL 2](https://docs.microsoft.com/en-us/windows/wsl/wsl2-install) version, +If you're running Windows 10, the [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and its latest [WSL 2](https://docs.microsoft.com/en-us/install-win10) version, support the installation of different Linux distributions, which include the Git and SSH clients. For current versions of Windows, you can also install the Git and SSH clients with diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 3978506cbf9..0425fe98d27 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -33,6 +33,9 @@ There are some differences in how a subscription applies, depending if you use G On a self-managed instance, a GitLab subscription provides the same set of features for all users. On GitLab.com you can apply a subscription to either a group or a personal namespace. +NOTE: **Note:** +Subscriptions cannot be transferred between GitLab.com and GitLab self-managed. A new subscription must be purchased and applied as needed. + ### Choosing a GitLab.com group or personal subscription On GitLab.com you can apply a subscription to either a group or a personal namespace. @@ -397,6 +400,15 @@ We recommend following these steps during renewal: TIP: **Tip:** You can find the _users over license_ in your instance's **Admin** dashboard by clicking on **{admin}** (**Admin Area**) in the top bar, or going to `/admin`. + The following table describes details of your admin dashboard and renewal terms: + + | Field | Description | + |:------|:------------| + | Users in License | The number of users you've paid for in the current license loaded on the system. This does not include the amount you've paid for `Users over license` during renewal. | + | Active users | The number of current active users on your system. | + | Maximum users | The highest number of active users on your system during the term of the loaded license. If this number exceeds your users in license count at any point, you incur users over license. | + | Users over license | The number of users that exceed the `Users in License` for the current license term. Charges for this number of users will be incurred at the next renewal. | + 1. Review your renewal details and complete the payment process. 1. A license for the renewal term will be available on the [Manage Purchases](https://customers.gitlab.com/subscriptions) page beneath your new subscription details. 1. [Upload](../user/admin_area/license.md) your new license to your instance. @@ -440,7 +452,7 @@ The new subscription tier is active when the license file is uploaded. ## Subscription expiry -When your subscription or trial expires, GitLab does not delete your data, but it may become inaccessible, depending on the tier at expiry. Some features may not behave as expected if you're not prepared for the expiry. For example, [environment specific variables not being passed](https://gitlab.com/gitlab-org/gitlab/issues/24759). +When your subscription or trial expires, GitLab does not delete your data, but it may become inaccessible, depending on the tier at expiry. Some features may not behave as expected if you're not prepared for the expiry. For example, [environment specific variables not being passed](https://gitlab.com/gitlab-org/gitlab/-/issues/24759). If you renew or upgrade, your data will again be accessible. @@ -508,7 +520,7 @@ Be aware that: ## Contact Support We also encourage all users to search our project trackers for known issues and -existing feature requests in the [GitLab](https://gitlab.com/gitlab-org/gitlab/issues/) project. +existing feature requests in the [GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/) project. These issues are the best avenue for getting updates on specific product plans and for communicating directly with the relevant GitLab team members. diff --git a/doc/system_hooks/system_hooks.md b/doc/system_hooks/system_hooks.md index 5ca8c0687f4..4ece58f533d 100644 --- a/doc/system_hooks/system_hooks.md +++ b/doc/system_hooks/system_hooks.md @@ -27,7 +27,7 @@ Your GitLab instance can perform HTTP POST requests on the following events: - `user_remove_from_group` - `user_update_for_group` -The triggers for most of these are self-explanatory, but `project_update` and `project_rename` deserve some clarification: `project_update` is fired any time an attribute of a project is changed (name, description, tags, etc.) *unless* the `path` attribute is also changed. In that case, a `project_rename` is triggered instead (so that, for instance, if all you care about is the repo URL, you can just listen for `project_rename`). +The triggers for most of these are self-explanatory, but `project_update` and `project_rename` deserve some clarification: `project_update` is fired any time an attribute of a project is changed (name, description, tags, etc.) *unless* the `path` attribute is also changed. In that case, a `project_rename` is triggered instead (so that, for instance, if all you care about is the repository URL, you can just listen for `project_rename`). `user_failed_login` is sent whenever a **blocked** user attempts to login and denied access. @@ -296,7 +296,7 @@ If the user is blocked via LDAP, `state` will be `ldap_blocked`. } ``` -`owner_name` and `owner_email` are always `null`. Please see <https://gitlab.com/gitlab-org/gitlab/issues/20011>. +`owner_name` and `owner_email` are always `null`. Please see <https://gitlab.com/gitlab-org/gitlab/-/issues/20011>. **Group removed:** @@ -313,7 +313,7 @@ If the user is blocked via LDAP, `state` will be `ldap_blocked`. } ``` -`owner_name` and `owner_email` are always `null`. Please see <https://gitlab.com/gitlab-org/gitlab/issues/20011>. +`owner_name` and `owner_email` are always `null`. Please see [issue #20011](https://gitlab.com/gitlab-org/gitlab/-/issues/20011). **Group renamed:** @@ -333,7 +333,7 @@ If the user is blocked via LDAP, `state` will be `ldap_blocked`. } ``` -`owner_name` and `owner_email` are always `null`. Please see <https://gitlab.com/gitlab-org/gitlab/issues/20011>. +`owner_name` and `owner_email` are always `null`. Please see <https://gitlab.com/gitlab-org/gitlab/-/issues/20011>. **New Group Member:** diff --git a/doc/topics/airgap/index.md b/doc/topics/airgap/index.md index 854e0103a69..3866ec50253 100644 --- a/doc/topics/airgap/index.md +++ b/doc/topics/airgap/index.md @@ -1,139 +1,3 @@ -# Offline GitLab - -Computers in an offline environment are isolated from the public internet as a security measure. This -page lists all the information available for running GitLab in an offline environment. - -## Quick start - -If you plan to deploy a GitLab instance on a physically-isolated and offline network, see the -[quick start guide](quick_start_guide.md) for configuration steps. - -## Features - -Follow these best practices to use GitLab's features in an offline environment: - -- [Operating the GitLab Secure scanners in an offline environment](../../user/application_security/offline_deployments/index.md). - -## Loading Docker images onto your offline host - -To use many GitLab features, including -[security scans](../../user/application_security/index.md#working-in-an-offline-environment) -and [Auto DevOps](../autodevops/), the GitLab Runner must be able to fetch the -relevant Docker images. - -The process for making these images available without direct access to the public internet -involves downloading the images then packaging and transferring them to the offline host. Here's an -example of such a transfer: - -1. Download Docker images from public internet. -1. Package Docker images as tar archives. -1. Transfer images to offline environment. -1. Load transferred images into offline Docker registry. - -### Using the official GitLab template - -GitLab provides a [vendored template](../../ci/yaml/README.md#includetemplate) -to ease this process. - -This template should be used in a new, empty project, with a `gitlab-ci.yml` file containing: - -```yaml -include: - - template: Secure-Binaries.gitlab-ci.yml -``` - -The pipeline downloads the Docker images needed for the Security Scanners and saves them as -[job artifacts](../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../user/packages/container_registry/index.md) -of the project where the pipeline is executed. These archives can be transferred to another location -and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon. -This method requires a GitLab Runner with access to both `gitlab.com` (including -`registry.gitlab.com`) and the local offline instance. This runner must run in -[privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) -to be able to use the `docker` command inside the jobs. This runner can be installed in a DMZ or on -a bastion, and used only for this specific project. - -#### Scheduling the updates - -By default, this project's pipeline will run only once, when the `.gitlab-ci.yml` is added to the -repo. To update the GitLab security scanners and signatures, it's necessary to run this pipeline -regularly. GitLab provides a way to [schedule pipelines](../../ci/pipelines/schedules.md). For -example, you can set this up to download and store the Docker images every week. - -Some images can be updated more frequently than others. For example, the [vulnerability database](https://hub.docker.com/r/arminc/clair-db/tags) -for Container Scanning is updated daily. To update this single image, create a new Scheduled -Pipeline that runs daily and set `SECURE_BINARIES_ANALYZERS` to `clair-vulnerabilities-db`. Only -this job will be triggered, and the image will be updated daily and made available in the project -registry. - -#### Using the secure bundle created - -The project using the `Secure-Binaries.gitlab-ci.yml` template should now host all the required -images and resources needed to run GitLab Security features. - -The next step is to tell the offline instance to use these resources instead of the default ones on -`gitlab.com`. This can be done by setting the right environment variables: -`SAST_ANALYZER_IMAGE_PREFIX` for SAST analyzers, `DS_ANALYZER_IMAGE_PREFIX` for Dependency Scanning, -and so on. - -You can set these variables in the project's `.gitlab-ci.yml` files by using the bundle directly, or -in the GitLab UI at the project or group level. See the [GitLab CI/CD environment variables page](../../ci/variables/README.md#custom-environment-variables) -for more information. - -#### Variables - -The following table shows which variables you can use with the `Secure-Binaries.gitlab-ci.yml` -template: - -| VARIABLE | Description | Default value | -|-------------------------------------------|-----------------------------------------------|-----------------------------------| -| `SECURE_BINARIES_ANALYZERS` | Comma-separated list of analyzers to download | `"bandit, brakeman, gosec, and so on..."` | -| `SECURE_BINARIES_DOWNLOAD_IMAGES` | Used to disable jobs | `"true"` | -| `SECURE_BINARIES_PUSH_IMAGES` | Push files to the project registry | `"true"` | -| `SECURE_BINARIES_SAVE_ARTIFACTS` | Also save image archives as artifacts | `"false"` | -| `SECURE_BINARIES_ANALYZER_VERSION` | Default analyzer version (docker tag) | `"2"` | - -### Alternate way without the official template - -If it's not possible to follow the above method, the images can be transferred manually instead: - -#### Example image packager script - -```shell -#!/bin/bash -set -ux - -# Specify needed analyzer images -analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} -gitlab=registry.gitlab.com/gitlab-org/security-products/analyzers/ - -for i in "${analyzers[@]}" -do - tarname="${i}_2.tar" - docker pull $gitlab$i:2 - docker save $gitlab$i:2 -o ./analyzers/${tarname} - chmod +r ./analyzers/${tarname} -done -``` - -#### Example image loader script - -This example loads the images from a bastion host to an offline host. In certain configurations, -physical media may be needed for such a transfer: - -```shell -#!/bin/bash -set -ux - -# Specify needed analyzer images -analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} -registry=$GITLAB_HOST:4567 - -for i in "${analyzers[@]}" -do - tarname="${i}_2.tar" - scp ./analyzers/${tarname} ${GITLAB_HOST}:~/${tarname} - ssh $GITLAB_HOST "sudo docker load -i ${tarname}" - ssh $GITLAB_HOST "sudo docker tag $(sudo docker images | grep $i | awk '{print $3}') ${registry}/analyzers/${i}:2" - ssh $GITLAB_HOST "sudo docker push ${registry}/analyzers/${i}:2" -done -``` +--- +redirect_to: '../offline/index.md' +--- diff --git a/doc/topics/authentication/index.md b/doc/topics/authentication/index.md index e4b86a39385..6ce5e203bbf 100644 --- a/doc/topics/authentication/index.md +++ b/doc/topics/authentication/index.md @@ -16,12 +16,9 @@ This page gathers all the resources for the topic **Authentication** within GitL ## GitLab administrators -- [LDAP (Community Edition)](../../administration/auth/ldap.md) -- [LDAP (Enterprise Edition)](../../administration/auth/ldap-ee.md) **(STARTER)** +- [LDAP](../../administration/auth/ldap/index.md) - [Enforce Two-factor Authentication (2FA)](../../security/two_factor_authentication.md#enforce-two-factor-authentication-2fa) - **Articles:** - - [How to Configure LDAP with GitLab CE](../../administration/auth/how_to_configure_ldap_gitlab_ce/index.md) - - [How to Configure LDAP with GitLab EE](../../administration/auth/how_to_configure_ldap_gitlab_ee/index.md) **(STARTER)** - [Feature Highlight: LDAP Integration](https://about.gitlab.com/blog/2014/07/10/feature-highlight-ldap-sync/) - [Debugging LDAP](https://about.gitlab.com/handbook/support/workflows/debugging_ldap.html) - **Integrations:** @@ -47,5 +44,5 @@ This page gathers all the resources for the topic **Authentication** within GitL - [Kanboard Plugin GitLab Authentication](https://github.com/kanboard/plugin-gitlab-auth) - [Jenkins GitLab OAuth Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+OAuth+Plugin) -- [How to customize GitLab to support OpenID authentication](http://eric.van-der-vlist.com/blog/2013/11/23/how-to-customize-gitlab-to-support-openid-authentication/) +- [How to customize GitLab to support OpenID authentication](https://blog.eric.van-der-vlist.com/2013/11/23/how-to-customize-gitlab-to-support-openid-authentication/) - [OKD - Configuring Authentication and User Agent](https://docs.okd.io/3.11/install_config/configuring_authentication.html#GitLab) diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 056b4c1caf4..253d5e56463 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -72,25 +72,20 @@ Avoid passing secrets as Docker build arguments if possible, as they may be persisted in your image. See [this discussion of best practices with secrets](https://github.com/moby/moby/issues/13490) for details. -## Passing secrets to `docker build` +## Forward CI variables to the build environment -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25514) in GitLab 12.3, but available in versions 11.9 and above. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25514) in GitLab 12.3, but available in versions 11.9 and above. -CI environment variables can be passed as -[build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) to the `docker build` command by listing them -by name, comma-separated, in the `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` -variable. For example, to forward the variables `CI_COMMIT_SHA` and `CI_ENVIRONMENT_NAME`, -set `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` to `CI_COMMIT_SHA,CI_ENVIRONMENT_NAME`. +CI variables can be forwarded into the build environment using the +`AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` CI variable. +The forwarded variables should be specified by name in a comma-separated +list. For example, to forward the variables `CI_COMMIT_SHA` and +`CI_ENVIRONMENT_NAME`, set `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` +to `CI_COMMIT_SHA,CI_ENVIRONMENT_NAME`. -CAUTION: **Caution:** -Unlike build arguments, these variables are not persisted by Docker in the final image, -though you can still persist them yourself. - -In projects: - -- Without a `Dockerfile`, these are available automatically as environment - variables. -- With a `Dockerfile`, the following is required: +- When using Buildpacks, the forwarded variables are available automatically + as environment variables. +- When using a `Dockerfile`, the following additional steps are required: 1. Activate the experimental `Dockerfile` syntax by adding the following code to the top of the file: @@ -128,7 +123,7 @@ repository or by specifying a project variable: ## Customize values for Helm Chart -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30628) in GitLab 12.6, `.gitlab/auto-deploy-values.yaml` will be used by default for Helm upgrades. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30628) in GitLab 12.6, `.gitlab/auto-deploy-values.yaml` will be used by default for Helm upgrades. You can override the default values in the `values.yaml` file in the [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app) by either: @@ -175,7 +170,7 @@ into your project and edit it as needed. ## Customizing the Kubernetes namespace -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27630) in GitLab 12.6. For clusters not managed by GitLab, you can customize the namespace in `.gitlab-ci.yml` by specifying @@ -302,13 +297,15 @@ applications. | `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. | | `AUTO_DEVOPS_ATOMIC_RELEASE` | As of GitLab 13.0, Auto DevOps uses [`--atomic`](https://v2.helm.sh/docs/helm/#options-43) for Helm deployments by default. Set this variable to `false` to disable the use of `--atomic` | | `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](stages.md#auto-build-using-cloud-native-buildpacks-beta). | +| `AUTO_DEVOPS_BUILD_IMAGE_CNB_BUILDER` | The builder used when building with Cloud Native Buildpacks. The default builder is `heroku/buildpacks:18`. [More details](stages.md#auto-build-using-cloud-native-buildpacks-beta). | | `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes won't prevent word splitting. [More details](#passing-arguments-to-docker-build). | -| `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#passing-secrets-to-docker-build) to be passed to the `docker build` command as secrets. | +| `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#forward-ci-variables-to-the-build-environment) to be forwarded to the build environment (the buildpack builder or `docker build`). | | `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/charts/auto-deploy-app). | | `AUTO_DEVOPS_CHART_REPOSITORY` | Helm Chart repository used to search for charts. Defaults to `https://charts.gitlab.io`. | | `AUTO_DEVOPS_CHART_REPOSITORY_NAME` | From GitLab 11.11, used to set the name of the Helm repository. Defaults to `gitlab`. | | `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | From GitLab 11.11, used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. | | `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | From GitLab 11.11, used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. | +| `AUTO_DEVOPS_DEPLOY_DEBUG` | From GitLab 13.1, if this variable is present, Helm will output debug logs. | | `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` | From GitLab 12.5, used in combination with [ModSecurity feature flag](../../user/clusters/applications.md#web-application-firewall-modsecurity) to toggle [ModSecurity's `SecRuleEngine`](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRuleEngine) behavior. Defaults to `DetectionOnly`. | | `BUILDPACK_URL` | Buildpack's full URL. Can point to either [a Git repository URL or a tarball URL](#custom-buildpacks). | | `CANARY_ENABLED` | From GitLab 11.0, used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments-premium). | @@ -368,7 +365,7 @@ The following table lists variables used to disable jobs. ### Application secret variables -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/49056) in GitLab 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49056) in GitLab 11.7. Some applications need to define secret variables that are accessible by the deployed application. Auto DevOps detects variables starting with `K8S_SECRET_`, and makes @@ -506,7 +503,7 @@ If you define `CANARY_ENABLED` in your project, such as setting `CANARY_ENABLED` ### Incremental rollout to production **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5415) in GitLab 10.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5415) in GitLab 10.8. TIP: **Tip:** You can also set this inside your [project's settings](index.md#deployment-strategy). @@ -563,7 +560,7 @@ removed in the future. ### Timed incremental rollout to production **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7545) in GitLab 11.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7545) in GitLab 11.4. TIP: **Tip:** You can also set this inside your [project's settings](index.md#deployment-strategy). @@ -597,7 +594,7 @@ The banner can be disabled for: - By an administrator running the following in a Rails console: ```ruby - Feature.get(:auto_devops_banner_disabled).enable + Feature.enable(:auto_devops_banner_disabled) ``` - Through the REST API with an admin access token: diff --git a/doc/topics/autodevops/img/guide_pipeline_stages_v12_3.png b/doc/topics/autodevops/img/guide_pipeline_stages_v12_3.png Binary files differdeleted file mode 100644 index b9bab112a9f..00000000000 --- a/doc/topics/autodevops/img/guide_pipeline_stages_v12_3.png +++ /dev/null diff --git a/doc/topics/autodevops/img/guide_pipeline_stages_v13_0.png b/doc/topics/autodevops/img/guide_pipeline_stages_v13_0.png Binary files differnew file mode 100644 index 00000000000..fb102879556 --- /dev/null +++ b/doc/topics/autodevops/img/guide_pipeline_stages_v13_0.png diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index e7165136cf0..767ea5ee7b7 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -1,6 +1,6 @@ # Auto DevOps -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/37115) in GitLab 10.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/37115) in GitLab 10.0. > - Generally available on GitLab 11.0. Auto DevOps provides pre-defined CI/CD configuration allowing you to automatically @@ -25,9 +25,11 @@ and GitLab does the rest, improving your productivity and efficiency. For an introduction to Auto DevOps, watch [AutoDevOps in GitLab 11.0](https://youtu.be/0Tc0YYBxqi4). +For requirements, see [Requirements for Auto DevOps](requirements.md) for more information. + ## Enabled by default -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41729) in GitLab 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41729) in GitLab 11.3. Auto DevOps is enabled by default for all projects and attempts to run on all pipelines in each project. An instance administrator can enable or disable this default in the @@ -35,7 +37,7 @@ in each project. An instance administrator can enable or disable this default in Auto DevOps automatically disables in individual projects on their first pipeline failure, if it has not been explicitly enabled for the project. -Since [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/issues/26655), Auto DevOps +Since [GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/issues/26655), Auto DevOps runs on pipelines automatically only if a [`Dockerfile` or matching buildpack](stages.md#auto-build) exists. @@ -54,7 +56,7 @@ configuring a cluster on GKE. After configuring the provider, you can follow the steps in the [quick start guide](quick_start_guide.md) to get started. In [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) and later, it is -possible to leverage Auto DevOps to deploy to [AWS ECS](#aws-ecs). +possible to leverage Auto DevOps to deploy to [AWS ECS](requirements.md#auto-devops-requirements-for-amazon-ecs). ## Comparison to application platforms and PaaS @@ -76,13 +78,14 @@ in multiple ways: ## Features -Comprised of a set of stages, Auto DevOps brings these best practices to your +Comprised of a set of [stages](stages.md), Auto DevOps brings these best practices to your project in a simple and automatic way: 1. [Auto Build](stages.md#auto-build) 1. [Auto Test](stages.md#auto-test) 1. [Auto Code Quality](stages.md#auto-code-quality-starter) **(STARTER)** 1. [Auto SAST (Static Application Security Testing)](stages.md#auto-sast-ultimate) **(ULTIMATE)** +1. [Auto Secret Detection](stages.md#auto-secret-detection-ultimate) **(ULTIMATE)** 1. [Auto Dependency Scanning](stages.md#auto-dependency-scanning-ultimate) **(ULTIMATE)** 1. [Auto License Compliance](stages.md#auto-license-compliance-ultimate) **(ULTIMATE)** 1. [Auto Container Scanning](stages.md#auto-container-scanning-ultimate) **(ULTIMATE)** @@ -111,127 +114,6 @@ NOTE: **Note** Kubernetes clusters can [be used without](../../user/project/clusters/index.md) Auto DevOps. -## Requirements - -### Kubernetes - -To make full use of Auto DevOps with Kubernetes, you need: - -- **Kubernetes** (for [Auto Review Apps](stages.md#auto-review-apps), - [Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring)) - - To enable deployments, you need: - - 1. A [Kubernetes 1.12+ cluster](../../user/project/clusters/index.md) for your - project. The easiest way is to create a - [new cluster using the GitLab UI](../../user/project/clusters/add_remove_clusters.md#create-new-cluster). - For Kubernetes 1.16+ clusters, you must perform additional configuration for - [Auto Deploy for Kubernetes 1.16+](stages.md#kubernetes-116). - 1. NGINX Ingress. You can deploy it to your Kubernetes cluster by installing - the [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), - after configuring GitLab's Kubernetes integration in the previous step. - - Alternatively, you can use the - [`nginx-ingress`](https://github.com/helm/charts/tree/master/stable/nginx-ingress) - Helm chart to install Ingress manually. - - NOTE: **Note:** - If you use your own Ingress instead of the one provided by GitLab's managed - apps, ensure you're running at least version 0.9.0 of NGINX Ingress and - [enable Prometheus metrics](https://github.com/helm/charts/tree/master/stable/nginx-ingress#prometheus-metrics) - for the response metrics to appear. You must also - [annotate](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) - the NGINX Ingress deployment to be scraped by Prometheus using - `prometheus.io/scrape: "true"` and `prometheus.io/port: "10254"`. - -- **Base domain** (for [Auto Review Apps](stages.md#auto-review-apps), - [Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring)) - - You need a domain configured with wildcard DNS, which all of your Auto DevOps - applications will use. If you're using the - [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), - the URL endpoint is automatically configured for you. - - You must also [specify the Auto DevOps base domain](#auto-devops-base-domain). - -- **GitLab Runner** (for all stages) - - Your Runner must be configured to run Docker, usually with either the - [Docker](https://docs.gitlab.com/runner/executors/docker.html) - or [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html) executors, with - [privileged mode enabled](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). - The Runners don't need to be installed in the Kubernetes cluster, but the - Kubernetes executor is easy to use and automatically autoscales. - You can configure Docker-based Runners to autoscale as well, using - [Docker Machine](https://docs.gitlab.com/runner/install/autoscaling.html). - - If you've configured GitLab's Kubernetes integration in the first step, you - can deploy it to your cluster by installing the - [GitLab-managed app for GitLab Runner](../../user/clusters/applications.md#gitlab-runner). - - Runners should be registered as [shared Runners](../../ci/runners/README.md#registering-a-shared-runner) - for the entire GitLab instance, or [specific Runners](../../ci/runners/README.md#registering-a-specific-runner) - that are assigned to specific projects (the default if you've installed the - GitLab Runner managed application). - -- **Prometheus** (for [Auto Monitoring](stages.md#auto-monitoring)) - - To enable Auto Monitoring, you need Prometheus installed either inside or - outside your cluster, and configured to scrape your Kubernetes cluster. - If you've configured GitLab's Kubernetes integration, you can deploy it to - your cluster by installing the - [GitLab-managed app for Prometheus](../../user/clusters/applications.md#prometheus). - - The [Prometheus service](../../user/project/integrations/prometheus.md) - integration must be enabled for the project, or enabled as a - [default service template](../../user/project/integrations/services_templates.md) - for the entire GitLab installation. - - To get response metrics (in addition to system metrics), you must - [configure Prometheus to monitor NGINX](../../user/project/integrations/prometheus_library/nginx_ingress.md#configuring-nginx-ingress-monitoring). - -- **cert-manager** (optional, for TLS/HTTPS) - - To enable HTTPS endpoints for your application, you must install cert-manager, - a native Kubernetes certificate management controller that helps with issuing - certificates. Installing cert-manager on your cluster issues a - [Let’s Encrypt](https://letsencrypt.org/) certificate and ensures the - certificates are valid and up-to-date. If you've configured GitLab's Kubernetes - integration, you can deploy it to your cluster by installing the - [GitLab-managed app for cert-manager](../../user/clusters/applications.md#cert-manager). - -If you don't have Kubernetes or Prometheus installed, then -[Auto Review Apps](stages.md#auto-review-apps), -[Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring) -are skipped. - -After all requirements are met, you can [enable Auto DevOps](#enablingdisabling-auto-devops). - -### AWS ECS - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) in GitLab 13.0. - -You can choose to target [AWS ECS](../../ci/cloud_deployment/index.md) as a deployment platform instead of using Kubernetes. - -To get started on Auto DevOps to ECS, you'll have to add a specific Environment -Variable. To do so, follow these steps: - -1. In your project, go to **Settings > CI / CD** and expand the **Variables** - section. - -1. Specify which AWS platform to target during the Auto DevOps deployment - by adding the `AUTO_DEVOPS_PLATFORM_TARGET` variable. - -1. Give this variable the value `ECS` before saving it. - -When you trigger a pipeline, if you have AutoDev Ops enabled and if you have correctly -[entered AWS credentials as environment variables](../../ci/cloud_deployment/index.md#deploy-your-application-to-aws-elastic-container-service-ecs), -your application will be deployed to AWS ECS. - -NOTE: **Note:** -If you have both a valid `AUTO_DEVOPS_PLATFORM_TARGET` variable and a Kubernetes cluster tied to your project, -only the deployment to Kubernetes will run. - ## Auto DevOps base domain The Auto DevOps base domain is required to use @@ -239,16 +121,18 @@ The Auto DevOps base domain is required to use [Auto Monitoring](stages.md#auto-monitoring). You can define the base domain in any of the following places: -- either under the cluster's settings, whether for +- either under the cluster's settings, whether for an instance, [projects](../../user/project/clusters/index.md#base-domain) or [groups](../../user/group/clusters/index.md#base-domain) -- or in instance-wide settings in **{admin}** **Admin Area > Settings** under the - **Continuous Integration and Delivery** section - or at the project level as a variable: `KUBE_INGRESS_BASE_DOMAIN` -- or at the group level as a variable: `KUBE_INGRESS_BASE_DOMAIN`. +- or at the group level as a variable: `KUBE_INGRESS_BASE_DOMAIN` +- or as an instance-wide fallback in **{admin}** **Admin Area > Settings** under the + **Continuous Integration and Delivery** section The base domain variable `KUBE_INGRESS_BASE_DOMAIN` follows the same order of precedence as other environment [variables](../../ci/variables/README.md#priority-of-environment-variables). +If the CI/CD variable is not set and the cluster setting is left blank, the instance-wide **Auto DevOps domain** +setting will be used if set. TIP: **Tip:** If you use the [GitLab managed app for Ingress](../../user/clusters/applications.md#ingress), @@ -256,9 +140,9 @@ the URL endpoint should be automatically configured for you. All you must do is use its value for the `KUBE_INGRESS_BASE_DOMAIN` variable. NOTE: **Note:** -`AUTO_DEVOPS_DOMAIN` was [deprecated in GitLab 11.8](https://gitlab.com/gitlab-org/gitlab-foss/issues/52363) +`AUTO_DEVOPS_DOMAIN` was [deprecated in GitLab 11.8](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52363) and replaced with `KUBE_INGRESS_BASE_DOMAIN`, and removed in -[GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-foss/issues/56959). +[GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/56959). Auto DevOps requires a wildcard DNS A record matching the base domain(s). For a base domain of `example.com`, you'd need a DNS entry like: @@ -305,7 +189,7 @@ After enabling the feature, an Auto DevOps pipeline is triggered on the default ### At the group level -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/52447) in GitLab 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/52447) in GitLab 11.10. Only administrators and group owners can enable or disable Auto DevOps at the group level. @@ -330,19 +214,9 @@ Auto DevOps at the group and project level, respectively. for Auto Deploy and Auto Review Apps to use. 1. Click **Save changes** for the changes to take effect. -### Enable for a percentage of projects - -You can use a feature flag to enable Auto DevOps by default to your desired percentage -of projects. From the console, enter the following command, replacing `10` with -your desired percentage: - -```ruby -Feature.get(:force_autodevops_on_by_default).enable_percentage_of_actors(10) -``` - ### Deployment strategy -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/38542) in GitLab 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/38542) in GitLab 11.0. You can change the deployment strategy used by Auto DevOps by going to your project's **{settings}** **Settings > CI/CD > Auto DevOps**. The following options @@ -362,13 +236,17 @@ are available: - `master` branch is directly deployed to staging. - Manual actions are provided for incremental rollout to production. +TIP: **Tip:** +Use the [blue-green deployment](../../ci/environments/incremental_rollouts.md#blue-green-deployment) technique +to minimize downtime and risk. + ## Using multiple Kubernetes clusters **(PREMIUM)** When using Auto DevOps, you can deploy different environments to different Kubernetes clusters, due to the 1:1 connection [existing between them](../../user/project/clusters/index.md#multiple-kubernetes-clusters-premium). -The [template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) +The [Deploy Job template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) used by Auto DevOps currently defines 3 environment names: - `review/` (every environment starting with `review/`) @@ -393,9 +271,6 @@ To add a different cluster for each environment: 1. Navigate to your project's **{cloud-gear}** **Operations > Kubernetes**. 1. Create the Kubernetes clusters with their respective environment scope, as described from the table above. - -  - 1. After creating the clusters, navigate to each cluster and install Helm Tiller and Ingress. Wait for the Ingress IP address to be assigned. 1. Make sure you've [configured your DNS](#auto-devops-base-domain) with the @@ -408,35 +283,6 @@ and verifying your application is deployed as a Review App in the Kubernetes cluster with the `review/*` environment scope. Similarly, you can check the other environments. -## Currently supported languages - -Note that not all buildpacks support Auto Test yet, as it's a relatively new -enhancement. All of Heroku's -[officially supported languages](https://devcenter.heroku.com/articles/heroku-ci#supported-languages) -support Auto Test. The languages supported by Heroku's Herokuish buildpacks all -support Auto Test, but notably the multi-buildpack does not. - -As of GitLab 10.0, the supported buildpacks are: - -```plaintext -- heroku-buildpack-multi v1.0.0 -- heroku-buildpack-ruby v168 -- heroku-buildpack-nodejs v99 -- heroku-buildpack-clojure v77 -- heroku-buildpack-python v99 -- heroku-buildpack-java v53 -- heroku-buildpack-gradle v23 -- heroku-buildpack-scala v78 -- heroku-buildpack-play v26 -- heroku-buildpack-php v122 -- heroku-buildpack-go v72 -- heroku-buildpack-erlang fa17af9 -- buildpack-nginx v8 -``` - -If your application needs a buildpack that is not in the above list, you -might want to use a [custom buildpack](customize.md#custom-buildpacks). - ## Limitations The following restrictions apply. @@ -490,11 +336,6 @@ The following are possible reasons: - No buildpack may exist for your application. Try specifying a [custom buildpack](customize.md#custom-buildpacks). -### Mismatch between testing frameworks - -Auto Test may fail because of a mismatch between testing frameworks. In this -case, you may need to customize your `.gitlab-ci.yml` with your test commands. - ### Pipeline that extends Auto DevOps with only / except fails If your pipeline fails with the following message: diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 859219689f9..ec5191dd4ac 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -152,8 +152,6 @@ these steps to enable Auto DevOps if it's disabled: After you save your changes, GitLab creates a new pipeline. To view it, go to **{rocket}** **CI/CD > Pipelines**. - - In the next section, we explain what each job does in the pipeline. ## Deploy the application @@ -167,7 +165,7 @@ without refreshing the page to **{status_success}** (for success) or The jobs are separated into stages: - + - **Build** - The application builds a Docker image and uploads it to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](stages.md#auto-build)). @@ -182,8 +180,9 @@ The jobs are separated into stages: - The `dependency_scanning` job checks if the application has any dependencies susceptible to vulnerabilities and is allowed to fail ([Auto Dependency Scanning](stages.md#auto-dependency-scanning-ultimate)) **(ULTIMATE)** - - The `sast` job runs static analysis on the current code to check for potential - security issues and is allowed to fail ([Auto SAST](stages.md#auto-sast-ultimate)) **(ULTIMATE)** + - Jobs suffixed with `-sast` run static analysis on the current code to check for potential + security issues, and are allowed to fail ([Auto SAST](stages.md#auto-sast-ultimate)) **(ULTIMATE)** + - The `secret-detection` job checks for leaked secrets and is allowed to fail ([Auto Secret Detection](stages.md#auto-secret-detection-ultimate)) **(ULTIMATE)** - The `license_management` job searches the application's dependencies to determine each of their licenses and is allowed to fail ([Auto License Compliance](stages.md#auto-license-compliance-ultimate)) **(ULTIMATE)** @@ -191,12 +190,17 @@ The jobs are separated into stages: NOTE: **Note:** All jobs except `test` are allowed to fail in the test stage. +- **Review** - Pipelines on `master` include this stage with a `dast_environment_deploy` job. + To learn more, see [Dynamic Application Security Testing (DAST)](../../user/application_security/dast/index.md). + - **Production** - After the tests and checks finish, the application deploys in Kubernetes ([Auto Deploy](stages.md#auto-deploy)). - **Performance** - Performance tests are run on the deployed application ([Auto Browser Performance Testing](stages.md#auto-browser-performance-testing-premium)). **(PREMIUM)** +- **Cleanup** - Pipelines on `master` include this stage with a `stop_dast_environment` job. + After running a pipeline, you should view your deployed website and learn how to monitor it. diff --git a/doc/topics/autodevops/requirements.md b/doc/topics/autodevops/requirements.md new file mode 100644 index 00000000000..b09a571fd16 --- /dev/null +++ b/doc/topics/autodevops/requirements.md @@ -0,0 +1,134 @@ +# Requirements for Auto DevOps + +You can set up Auto DevOps for [Kubernetes](#auto-devops-requirements-for-kubernetes) +or [Amazon Elastic Container Service (ECS)](#auto-devops-requirements-for-amazon-ecs). +For more information about Auto DevOps, see [the main Auto DevOps page](index.md) +or the [quick start guide](quick_start_guide.md). + +## Auto DevOps requirements for Kubernetes + +To make full use of Auto DevOps with Kubernetes, you need: + +- **Kubernetes** (for [Auto Review Apps](stages.md#auto-review-apps), + [Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring)) + + To enable deployments, you need: + + 1. A [Kubernetes 1.12+ cluster](../../user/project/clusters/index.md) for your + project. The easiest way is to create a + [new cluster using the GitLab UI](../../user/project/clusters/add_remove_clusters.md#create-new-cluster). + For Kubernetes 1.16+ clusters, you must perform additional configuration for + [Auto Deploy for Kubernetes 1.16+](stages.md#kubernetes-116). + 1. NGINX Ingress. You can deploy it to your Kubernetes cluster by installing + the [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), + after configuring GitLab's Kubernetes integration in the previous step. + + Alternatively, you can use the + [`nginx-ingress`](https://github.com/helm/charts/tree/master/stable/nginx-ingress) + Helm chart to install Ingress manually. + + NOTE: **Note:** + If you use your own Ingress instead of the one provided by GitLab's managed + apps, ensure you're running at least version 0.9.0 of NGINX Ingress and + [enable Prometheus metrics](https://github.com/helm/charts/tree/master/stable/nginx-ingress#prometheus-metrics) + for the response metrics to appear. You must also + [annotate](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) + the NGINX Ingress deployment to be scraped by Prometheus using + `prometheus.io/scrape: "true"` and `prometheus.io/port: "10254"`. + +- **Base domain** (for [Auto Review Apps](stages.md#auto-review-apps), + [Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring)) + + You need a domain configured with wildcard DNS, which all of your Auto DevOps + applications will use. If you're using the + [GitLab-managed app for Ingress](../../user/clusters/applications.md#ingress), + the URL endpoint is automatically configured for you. + + You must also [specify the Auto DevOps base domain](index.md#auto-devops-base-domain). + +- **GitLab Runner** (for all stages) + + Your Runner must be configured to run Docker, usually with either the + [Docker](https://docs.gitlab.com/runner/executors/docker.html) + or [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html) executors, with + [privileged mode enabled](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). + The Runners don't need to be installed in the Kubernetes cluster, but the + Kubernetes executor is easy to use and automatically autoscales. + You can configure Docker-based Runners to autoscale as well, using + [Docker Machine](https://docs.gitlab.com/runner/install/autoscaling.html). + + If you've configured GitLab's Kubernetes integration in the first step, you + can deploy it to your cluster by installing the + [GitLab-managed app for GitLab Runner](../../user/clusters/applications.md#gitlab-runner). + + Runners should be registered as [shared Runners](../../ci/runners/README.md#shared-runners) + for the entire GitLab instance, or [specific Runners](../../ci/runners/README.md#specific-runners) + that are assigned to specific projects (the default if you've installed the + GitLab Runner managed application). + +- **Prometheus** (for [Auto Monitoring](stages.md#auto-monitoring)) + + To enable Auto Monitoring, you need Prometheus installed either inside or + outside your cluster, and configured to scrape your Kubernetes cluster. + If you've configured GitLab's Kubernetes integration, you can deploy it to + your cluster by installing the + [GitLab-managed app for Prometheus](../../user/clusters/applications.md#prometheus). + + The [Prometheus service](../../user/project/integrations/prometheus.md) + integration must be enabled for the project, or enabled as a + [default service template](../../user/project/integrations/services_templates.md) + for the entire GitLab installation. + + To get response metrics (in addition to system metrics), you must + [configure Prometheus to monitor NGINX](../../user/project/integrations/prometheus_library/nginx_ingress.md#configuring-nginx-ingress-monitoring). + +- **cert-manager** (optional, for TLS/HTTPS) + + To enable HTTPS endpoints for your application, you must install cert-manager, + a native Kubernetes certificate management controller that helps with issuing + certificates. Installing cert-manager on your cluster issues a + [Let’s Encrypt](https://letsencrypt.org/) certificate and ensures the + certificates are valid and up-to-date. If you've configured GitLab's Kubernetes + integration, you can deploy it to your cluster by installing the + [GitLab-managed app for cert-manager](../../user/clusters/applications.md#cert-manager). + +If you don't have Kubernetes or Prometheus installed, then +[Auto Review Apps](stages.md#auto-review-apps), +[Auto Deploy](stages.md#auto-deploy), and [Auto Monitoring](stages.md#auto-monitoring) +are skipped. + +After all requirements are met, you can [enable Auto DevOps](index.md#enablingdisabling-auto-devops). + +## Auto DevOps requirements for Amazon ECS + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) in GitLab 13.0. + +You can choose to target [Amazon Elastic Container Service (ECS)](../../ci/cloud_deployment/index.md) as a deployment platform instead of using Kubernetes. + +To get started on Auto DevOps to Amazon ECS, you'll have to add a specific Environment +Variable. To do so, follow these steps: + +1. In your project, go to **Settings > CI / CD** and expand the **Variables** + section. + +1. Specify which AWS platform to target during the Auto DevOps deployment + by adding the `AUTO_DEVOPS_PLATFORM_TARGET` variable. + +1. Give this variable the value `ECS` before saving it. + +When you trigger a pipeline, if Auto DevOps is enabled and if you've correctly +[entered AWS credentials as environment variables](../../ci/cloud_deployment/index.md#deploy-your-application-to-the-aws-elastic-container-service-ecs), +your application will be deployed to Amazon ECS. + +NOTE: **Note:** +If you have both a valid `AUTO_DEVOPS_PLATFORM_TARGET` variable and a Kubernetes cluster tied to your project, +only the deployment to Kubernetes will run. + +CAUTION: **Warning:** +Setting the `AUTO_DEVOPS_PLATFORM_TARGET` variable to `ECS` will trigger jobs +defined in the [`Jobs/Deploy/ECS.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml). +However, it's not recommended to [include](../../ci/yaml/README.md#includetemplate) +it on its own. This template is designed to be used with Auto DevOps only. It may change +unexpectedly causing your pipeline to fail if included on its own. Also, the job +names within this template may also change. Don't override these jobs' names in your +own pipeline, as the override will stop working when the name changes. diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 8c56a87ba30..0c7c4919431 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -53,7 +53,9 @@ troubleshoot. Auto Build supports building your application using [Cloud Native Buildpacks](https://buildpacks.io) through the [`pack` command](https://github.com/buildpacks/pack). To use Cloud Native Buildpacks, -set the CI variable `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` to a non-empty value. +set the CI variable `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` to a non-empty +value. The default builder is `heroku/buildpacks:18` but a different builder +can be selected using the CI variable `AUTO_DEVOPS_BUILD_IMAGE_CNB_BUILDER`. Cloud Native Buildpacks (CNBs) are an evolution of Heroku buildpacks, and will eventually supersede Herokuish-based builds within Auto DevOps. For more @@ -82,11 +84,39 @@ Auto Test runs the appropriate tests for your application using your project to detect the language and framework. Several languages and frameworks are detected automatically, but if your language is not detected, you may be able to create a [custom buildpack](customize.md#custom-buildpacks). -Check the [currently supported languages](index.md#currently-supported-languages). +Check the [currently supported languages](#currently-supported-languages). Auto Test uses tests you already have in your application. If there are no tests, it's up to you to add them. +### Currently supported languages + +Note that not all buildpacks support Auto Test yet, as it's a relatively new +enhancement. All of Heroku's +[officially supported languages](https://devcenter.heroku.com/articles/heroku-ci#supported-languages) +support Auto Test. The languages supported by Heroku's Herokuish buildpacks all +support Auto Test, but notably the multi-buildpack does not. + +The supported buildpacks are: + +```plaintext +- heroku-buildpack-multi +- heroku-buildpack-ruby +- heroku-buildpack-nodejs +- heroku-buildpack-clojure +- heroku-buildpack-python +- heroku-buildpack-java +- heroku-buildpack-gradle +- heroku-buildpack-scala +- heroku-buildpack-play +- heroku-buildpack-php +- heroku-buildpack-go +- buildpack-nginx +``` + +If your application needs a buildpack that is not in the above list, you +might want to use a [custom buildpack](customize.md#custom-buildpacks). + ## Auto Code Quality **(STARTER)** Auto Code Quality uses the @@ -114,6 +144,22 @@ warnings. To learn more about [how SAST works](../../user/application_security/sast/index.md), see the documentation. +## Auto Secret Detection **(ULTIMATE)** + +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. + +Secret Detection uses the +[Secret Detection Docker image](https://gitlab.com/gitlab-org/security-products/analyzers/secrets) to run Secret Detection on the current code, and checks for leaked secrets. The +Auto Secret Detection stage runs only on the +[Ultimate](https://about.gitlab.com/pricing/) tier, and requires +[GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or above. + +After creating the report, it's uploaded as an artifact which you can later +download and evaluate. The merge request widget also displays any security +warnings. + +To learn more, see [Secret Detection](../../user/application_security/secret_detection/index.md). + ## Auto Dependency Scanning **(ULTIMATE)** > Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. @@ -166,7 +212,7 @@ see the documentation. ## Auto Review Apps This is an optional step, since many projects don't have a Kubernetes cluster -available. If the [requirements](index.md#requirements) are not met, the job is +available. If the [requirements](requirements.md) are not met, the job is silently skipped. [Review Apps](../../ci/review_apps/index.md) are temporary application environments based on the @@ -267,7 +313,7 @@ Any performance differences between the source and target branches are also ## Auto Deploy This is an optional step, since many projects don't have a Kubernetes cluster -available. If the [requirements](index.md#requirements) are not met, the job is skipped. +available. If the [requirements](requirements.md) are not met, the job is skipped. After a branch or merge request is merged into the project's default branch (usually `master`), Auto Deploy deploys the application to a `production` environment in @@ -328,7 +374,7 @@ as it attempts to fetch the image using `CI_REGISTRY_PASSWORD`. CAUTION: **Deprecation** The default value for the `deploymentApiVersion` setting was changed from -`extensions/v1beta` to `apps/v1` in [GitLab 13.0](https://gitlab.com/gitlab-org/charts/auto-deploy-app/issues/47). +`extensions/v1beta` to `apps/v1` in [GitLab 13.0](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/issues/47). In Kubernetes 1.16 and later, a number of [APIs were removed](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/), @@ -467,7 +513,7 @@ traffic within a local namespace, and from the `gitlab-managed-apps` namespace. All other inbound connections are blocked. Outbound traffic (for example, to the Internet) is not affected by the default policy. -You can also provide a custom [policy specification](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#networkpolicyspec-v1-networking-k8s-io) +You can also provide a custom [policy specification](https://kubernetes.io/docs/concepts/services-networking/network-policies/) in the `.gitlab/auto-deploy-values.yaml` file, for example: ```yaml @@ -568,7 +614,7 @@ GitLab provides some initial alerts for you after you install Prometheus: To use Auto Monitoring: -1. [Install and configure the requirements](index.md#requirements). +1. [Install and configure the Auto DevOps requirements](requirements.md). 1. [Enable Auto DevOps](index.md#enablingdisabling-auto-devops), if you haven't done already. 1. Navigate to your project's **{rocket}** **CI/CD > Pipelines** and click **Run Pipeline**. 1. After the pipeline finishes successfully, open the diff --git a/doc/topics/git/feature_branch_development.md b/doc/topics/git/feature_branch_development.md new file mode 100644 index 00000000000..ab3adf54dd7 --- /dev/null +++ b/doc/topics/git/feature_branch_development.md @@ -0,0 +1,86 @@ +--- +type: how-tos +--- + +# Develop on a feature branch + +GitLab values encourage the use of [Minimal Viable Change (MVC)](https://about.gitlab.com/handbook/values/#minimal-viable-change-mvc). +However, viable changes are not always small. In such cases, it can help to set up a dedicated feature branch. +People can contribute MRs to that feature branch, without affecting the functionality of the default (usually `master`) branch. + +Once work on the development branch is complete, then the feature branch can be finally merged into the default branch. + +GitLab frequently implements this process whenever there is an MVC that requires multiple MRs. + +## Use case: GitLab's release posts + +This section describes the use case with GitLab [release posts](https://about.gitlab.com/handbook/marketing/blog/release-posts/). +Dozens of GitLab team members contribute to each monthly release post. +In such cases, it may be more efficient to submit an MR on the release post feature branch instead of master. + +In this case, the feature branch would be `release-X-Y`. Assuming the `release-X-Y` branch already exists, you can set up an MR against that branch, with the following steps: + +1. Create a new branch (`test-branch`) against the feature branch (`release-X-Y`): + + ```shell + git checkout -b test-branch release-X-Y + ``` + + You should now be on a branch named `test-branch`. + +1. Make desired changes on the `test-branch`. +1. Add your changes, commit, and push to the `test-branch`: + + ```shell + git add . + ``` + +1. Commit your changes: + + ```shell + git commit -m "Some good reason" + ``` + +1. Push your changes to the repository: + + ```shell + git push --set-upstream origin test-branch + ``` + +1. Navigate to the URL for your repository. In this case, the repository is `www-gitlab-com`, available at `https://gitlab.com/gitlab-com/www-gitlab-com`. + + If needed, sign in to GitLab. You should then see an option to **Create merge request**: + +  + +1. After you click **Create merge request**, you'll see an option to **Change branches**. Select that option. + +1. In the **New Merge Request** screen, you can now select the **Source** and **Target** branches. +In the screenshot shown, +we have selected `test-branch` as the source, and `release-13-0` as the target. + +  + +1. Once you've selected the Source and Target branches, click **Compare branches and continue**. + You should see an entry similar to: + + ```plaintext + New Merge Request + + From test-branch into release-13-0 + ``` + + An entry like this confirms that your MR will **not** merge into master. + +1. Make any additional changes in the **New Merge Request** screen, and click **Submit merge request**. +1. In the new merge request, look for **Request to merge**. You'll see an entry similar to: + + ```plaintext + Request to merge test-branch into release-13-0 + ``` + + That confirms you've set up the MR to merge into the specified branch, not master. + +1. Proceed with the change as you would with any other MR. +1. When your MR is approved, and an appropriate user merges that MR, you can rest assured that your work is incorporated directly into the feature branch. +When the feature branch is ready, it can then be merged into master. diff --git a/doc/topics/git/img/create_merge_request_v13_1.png b/doc/topics/git/img/create_merge_request_v13_1.png Binary files differnew file mode 100644 index 00000000000..a725149f6a2 --- /dev/null +++ b/doc/topics/git/img/create_merge_request_v13_1.png diff --git a/doc/topics/git/img/modify_branches_v13_1.png b/doc/topics/git/img/modify_branches_v13_1.png Binary files differnew file mode 100644 index 00000000000..dc517dd249f --- /dev/null +++ b/doc/topics/git/img/modify_branches_v13_1.png diff --git a/doc/topics/git/index.md b/doc/topics/git/index.md index 9e6875312f3..2e36fea14bf 100644 --- a/doc/topics/git/index.md +++ b/doc/topics/git/index.md @@ -68,6 +68,7 @@ If you have problems with Git, the following may help: ## Branching strategies - [Feature branch workflow](../../gitlab-basics/feature_branch_workflow.md) +- [Develop on a feature branch](feature_branch_development.md) - [GitLab Flow](../gitlab_flow.md) - [Git Branching - Branches in a Nutshell](https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell) - [Git Branching - Branching Workflows](https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows) @@ -91,7 +92,7 @@ Git-related queries from GitLab. The following relate to Git Large File Storage: - [Getting Started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) -- [Migrate an existing Git repo with Git LFS](lfs/migrate_to_git_lfs.md) +- [Migrate an existing Git repository with Git LFS](lfs/migrate_to_git_lfs.md) - [Removing objects from LFS](lfs/index.md#removing-objects-from-lfs) - [GitLab Git LFS user documentation](lfs/index.md) - [GitLab Git LFS admin documentation](../../administration/lfs/index.md) diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index 33b7fa45691..706d3c3eddf 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -37,7 +37,7 @@ Documentation for GitLab instance administrators is under [LFS administration do - Any Git LFS request will ask for HTTPS credentials to be provided so a good Git credentials store is recommended - Git LFS always assumes HTTPS so if you have GitLab server on HTTP you will have - to add the URL to Git config manually (see [troubleshooting](#troubleshooting)) + to add the URL to Git configuration manually (see [troubleshooting](#troubleshooting)) NOTE: **Note:** With 8.12 GitLab added LFS support to SSH. The Git LFS communication @@ -83,7 +83,7 @@ git clone git@gitlab.example.com:group/project.git ``` If you already cloned the repository and you want to get the latest LFS object -that are on the remote repository, eg. for a branch from origin: +that are on the remote repository, such as for a branch from origin: ```shell git lfs fetch origin master @@ -91,18 +91,18 @@ git lfs fetch origin master ### Migrate an existing repo to Git LFS -Read the documentation on how to [migrate an existing Git repo with Git LFS](migrate_to_git_lfs.md). +Read the documentation on how to [migrate an existing Git repository with Git LFS](migrate_to_git_lfs.md). ### Removing objects from LFS To remove objects from LFS: -1. Use [BFG-Cleaner](../../../user/project/repository/reducing_the_repo_size_using_git.md#using-the-bfg-repo-cleaner) or [filter-branch](../../../user/project/repository/reducing_the_repo_size_using_git.md#using-git-filter-branch) to remove the objects from the repository. +1. Use [`git filter-repo`](../../../user/project/repository/reducing_the_repo_size_using_git.md) to remove the objects from the repository. 1. Delete the relevant LFS lines for the objects you have removed from your `.gitattributes` file and commit those changes. ## File Locking -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/35856) in GitLab 10.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/35856) in GitLab 10.5. The first thing to do before using File Locking is to tell Git LFS which kind of files are lockable. The following command will store PNG files @@ -128,7 +128,7 @@ in order to do that you can edit the `.gitattributes` file manually: ``` After a file type has been registered as lockable, Git LFS will make -them readonly on the file system automatically. This means you will +them read-only on the file system automatically. This means you will need to lock the file before editing it. ### Managing Locked Files @@ -205,8 +205,8 @@ If the status `error 501` is shown, it is because: on how to enable LFS support. - Git LFS client version is not supported by GitLab server. Check your Git LFS - version with `git lfs version`. Check the Git config of the project for traces - of deprecated API with `git lfs -l`. If `batch = false` is set in the config, + version with `git lfs version`. Check the Git configuration of the project for traces + of deprecated API with `git lfs -l`. If `batch = false` is set in the configuration, remove the line and try to update your Git LFS client. Only version 1.0.1 and newer are supported. @@ -218,9 +218,9 @@ the LFS client is trying to reach GitLab through HTTPS. However, your GitLab instance is being served on HTTP. This behavior is caused by Git LFS using HTTPS connections by default when a -`lfsurl` is not set in the Git config. +`lfsurl` is not set in the Git configuration. -To prevent this from happening, set the lfs URL in project Git config: +To prevent this from happening, set the LFS URL in project Git configuration: ```shell git config --add lfs.url "http://gitlab.example.com/group/project.git/info/lfs" diff --git a/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md b/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md index 05b749d7b24..09087fcae13 100644 --- a/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md @@ -1,7 +1,7 @@ # Migration guide from Git Annex to Git LFS >**Note:** -Git Annex support [has been removed](https://gitlab.com/gitlab-org/gitlab/issues/1648) in GitLab Enterprise +Git Annex support [has been removed](https://gitlab.com/gitlab-org/gitlab/-/issues/1648) in GitLab Enterprise Edition 9.0 (2017/03/22). Both [Git Annex](http://git-annex.branchable.com/) and [Git LFS](https://git-lfs.github.com/) are tools to manage large files in Git. @@ -172,7 +172,7 @@ GitLab.com), therefore, you don't need to do anything server-side. If the terminal doesn't prompt you with a full response on `git-lfs` commands, [install the Git LFS client](https://git-lfs.github.com/) first. -1. Inside the repo, run the following command to initiate LFS: +1. Inside the repository, run the following command to initiate LFS: ```shell git lfs install @@ -189,7 +189,7 @@ GitLab.com), therefore, you don't need to do anything server-side. ``` Once you do that, run `git status` and you'll see `.gitattributes` added - to your repo. It collects all file patterns that you chose to track via + to your repository. It collects all file patterns that you chose to track via `git-lfs`. 1. Add the files, commit and push them to GitLab: diff --git a/doc/topics/git/lfs/migrate_to_git_lfs.md b/doc/topics/git/lfs/migrate_to_git_lfs.md index 60859686047..a64639a9238 100644 --- a/doc/topics/git/lfs/migrate_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_to_git_lfs.md @@ -16,18 +16,18 @@ the files are still referenced by previous commits. Through the method described on this document, first migrate to Git LFS with a tool such as the open source community-maintained [BFG](https://rtyley.github.io/bfg-repo-cleaner/) -through a mirror repo, then clean up the repository's history, +through a mirror repository, then clean up the repository's history, and lastly create LFS tracking rules to prevent new binary files from being added. This tutorial was inspired by the guide -[Use BFG to migrate a repo to Git LFS](https://confluence.atlassian.com/bitbucket/use-bfg-to-migrate-a-repo-to-git-lfs-834233484.html). +[Use BFG to migrate a repository to Git LFS](https://confluence.atlassian.com/bitbucket/use-bfg-to-migrate-a-repo-to-git-lfs-834233484.html). For more information on Git LFS, see the [references](#references) below. CAUTION: **Warning:** The method described on this guide rewrites Git history. Make -sure to back up your repo before beginning and use it at your +sure to back up your repository before beginning and use it at your own risk. ## Requirements @@ -71,7 +71,7 @@ Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs- Create a copy of your repository so that you can recover it in case something goes wrong. -1. Clone `--mirror` the repo: +1. Clone `--mirror` the repository: Cloning with the mirror flag will create a bare repository. This ensures you get all the branches within the repo. @@ -102,7 +102,7 @@ Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs- git reflog expire --expire=now --all && git gc --prune=now --aggressive ``` - You can also take a look on how to further [clean the repo](../../../user/project/repository/reducing_the_repo_size_using_git.md), + You can also take a look on how to further [clean the repository](../../../user/project/repository/reducing_the_repo_size_using_git.md), but it's not necessary for the purposes of this guide. 1. Install Git LFS in the mirror repository: @@ -166,7 +166,7 @@ but commented out to help encourage others to add to it in the future. --> - [Migrate from Git Annex to Git LFS](migrate_from_git_annex_to_git_lfs.md) - [GitLab's Git LFS user documentation](index.md) - [GitLab's Git LFS administrator documentation](../../../administration/lfs/index.md) -- Alternative method to [migrate an existing repo to Git LFS](https://github.com/git-lfs/git-lfs/wiki/Tutorial#migrating-existing-repository-data-to-lfs) +- Alternative method to [migrate an existing repository to Git LFS](https://github.com/git-lfs/git-lfs/wiki/Tutorial#migrating-existing-repository-data-to-lfs) <!-- Test project: diff --git a/doc/topics/git/numerous_undo_possibilities_in_git/index.md b/doc/topics/git/numerous_undo_possibilities_in_git/index.md index 8597325db7b..fdf86d8f646 100644 --- a/doc/topics/git/numerous_undo_possibilities_in_git/index.md +++ b/doc/topics/git/numerous_undo_possibilities_in_git/index.md @@ -26,6 +26,11 @@ This means that until Git automatically cleans detached commits (which cannot be accessed by branch or tag) it will be possible to view them with `git reflog` command and access them with direct commit ID. Read more about _[redoing the undo](#redoing-the-undo)_ in the section below. +> For more information about working with Git and GitLab: +> +> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Learn why [North Western Mutual chose GitLab](https://youtu.be/kPNMyxKRRoM) for their Enterprise source code management. +> - Learn how to [get started with Git](https://about.gitlab.com/resources/whitepaper-moving-to-git/). + ## Introduction This guide is organized depending on the [stage of development](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository) diff --git a/doc/topics/git/partial_clone.md b/doc/topics/git/partial_clone.md index 46318a7f30d..7462406cad3 100644 --- a/doc/topics/git/partial_clone.md +++ b/doc/topics/git/partial_clone.md @@ -141,7 +141,7 @@ enabled on the Git server: 1. **Create a new Git repository and fetch.** Support for `--filter=sparse:oid` using the clone command is incomplete, so we will emulate the clone command by hand, using `git init` and `git fetch`. Follow - [issue tracking support for `--filter=sparse:oid`](https://gitlab.com/gitlab-org/git/issues/4) + [issue tracking support for `--filter=sparse:oid`](https://gitlab.com/gitlab-org/git/-/issues/4) for updates. ```shell @@ -173,7 +173,7 @@ enabled on the Git server: 1. **Sparse checkout** must be enabled and configured to prevent objects from other paths being downloaded automatically when checking out branches. Follow - [issue proposing automating sparse checkouts](https://gitlab.com/gitlab-org/git/issues/5) for updates. + [issue proposing automating sparse checkouts](https://gitlab.com/gitlab-org/git/-/issues/5) for updates. ```shell # Enable sparse checkout diff --git a/doc/topics/gitlab_flow.md b/doc/topics/gitlab_flow.md index 4738c5b23d7..6382ac0957a 100644 --- a/doc/topics/gitlab_flow.md +++ b/doc/topics/gitlab_flow.md @@ -15,7 +15,7 @@ Organizations coming to Git from other version control systems frequently find i This article describes GitLab flow, which integrates the Git workflow with an issue tracking system. It offers a simple, transparent, and effective way to work with Git. - + When converting to Git, you have to get used to the fact that it takes three steps to share a commit with colleagues. Most version control systems have only one step: committing from the working copy to a shared server. diff --git a/doc/topics/index.md b/doc/topics/index.md index e2749b58b03..634dd70613a 100644 --- a/doc/topics/index.md +++ b/doc/topics/index.md @@ -14,5 +14,6 @@ tutorials, technical overviews, blog posts) and videos. - [GitLab Flow](gitlab_flow.md) - [GitLab Installation](../install/README.md) - [GitLab Pages](../user/project/pages/index.md) +- [Offline GitLab](offline/index.md) >**Note:** More topics will be available soon. diff --git a/doc/topics/offline/index.md b/doc/topics/offline/index.md new file mode 100644 index 00000000000..6d4c486d350 --- /dev/null +++ b/doc/topics/offline/index.md @@ -0,0 +1,138 @@ +# Offline GitLab + +Computers in an offline environment are isolated from the public internet as a security measure. This +page lists all the information available for running GitLab in an offline environment. + +## Quick start + +If you plan to deploy a GitLab instance on a physically-isolated and offline network, see the +[quick start guide](quick_start_guide.md) for configuration steps. + +## Features + +Follow these best practices to use GitLab's features in an offline environment: + +- [Operating the GitLab Secure scanners in an offline environment](../../user/application_security/offline_deployments/index.md). + +## Loading Docker images onto your offline host + +To use many GitLab features, including +[security scans](../../user/application_security/index.md#working-in-an-offline-environment) +and [Auto DevOps](../autodevops/), the GitLab Runner must be able to fetch the +relevant Docker images. + +The process for making these images available without direct access to the public internet +involves downloading the images then packaging and transferring them to the offline host. Here's an +example of such a transfer: + +1. Download Docker images from public internet. +1. Package Docker images as tar archives. +1. Transfer images to offline environment. +1. Load transferred images into offline Docker registry. + +### Using the official GitLab template + +GitLab provides a [vendored template](../../ci/yaml/README.md#includetemplate) +to ease this process. + +This template should be used in a new, empty project, with a `gitlab-ci.yml` file containing: + +```yaml +include: + - template: Secure-Binaries.gitlab-ci.yml +``` + +The pipeline downloads the Docker images needed for the Security Scanners and saves them as +[job artifacts](../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../user/packages/container_registry/index.md) +of the project where the pipeline is executed. These archives can be transferred to another location +and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon. +This method requires a GitLab Runner with access to both `gitlab.com` (including +`registry.gitlab.com`) and the local offline instance. This runner must run in +[privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) +to be able to use the `docker` command inside the jobs. This runner can be installed in a DMZ or on +a bastion, and used only for this specific project. + +#### Scheduling the updates + +By default, this project's pipeline will run only once, when the `.gitlab-ci.yml` is added to the +repo. To update the GitLab security scanners and signatures, it's necessary to run this pipeline +regularly. GitLab provides a way to [schedule pipelines](../../ci/pipelines/schedules.md). For +example, you can set this up to download and store the Docker images every week. + +Some images can be updated more frequently than others. For example, the [vulnerability database](https://hub.docker.com/r/arminc/clair-db/tags) +for Container Scanning is updated daily. To update this single image, create a new Scheduled +Pipeline that runs daily and set `SECURE_BINARIES_ANALYZERS` to `clair-vulnerabilities-db`. Only +this job will be triggered, and the image will be updated daily and made available in the project +registry. + +#### Using the secure bundle created + +The project using the `Secure-Binaries.gitlab-ci.yml` template should now host all the required +images and resources needed to run GitLab Security features. + +Next, you must tell the offline instance to use these resources instead of the default ones on +GitLab.com. To do so, set the environment variable `SECURE_ANALYZERS_PREFIX` with the URL of the +project [container registry](../../user/packages/container_registry/index.md). + +You can set this variable in the projects' `.gitlab-ci.yml`, or +in the GitLab UI at the project or group level. See the [GitLab CI/CD environment variables page](../../ci/variables/README.md#custom-environment-variables) +for more information. + +#### Variables + +The following table shows which variables you can use with the `Secure-Binaries.gitlab-ci.yml` +template: + +| VARIABLE | Description | Default value | +|-------------------------------------------|-----------------------------------------------|-----------------------------------| +| `SECURE_BINARIES_ANALYZERS` | Comma-separated list of analyzers to download | `"bandit, brakeman, gosec, and so on..."` | +| `SECURE_BINARIES_DOWNLOAD_IMAGES` | Used to disable jobs | `"true"` | +| `SECURE_BINARIES_PUSH_IMAGES` | Push files to the project registry | `"true"` | +| `SECURE_BINARIES_SAVE_ARTIFACTS` | Also save image archives as artifacts | `"false"` | +| `SECURE_BINARIES_ANALYZER_VERSION` | Default analyzer version (Docker tag) | `"2"` | + +### Alternate way without the official template + +If it's not possible to follow the above method, the images can be transferred manually instead: + +#### Example image packager script + +```shell +#!/bin/bash +set -ux + +# Specify needed analyzer images +analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} +gitlab=registry.gitlab.com/gitlab-org/security-products/analyzers/ + +for i in "${analyzers[@]}" +do + tarname="${i}_2.tar" + docker pull $gitlab$i:2 + docker save $gitlab$i:2 -o ./analyzers/${tarname} + chmod +r ./analyzers/${tarname} +done +``` + +#### Example image loader script + +This example loads the images from a bastion host to an offline host. In certain configurations, +physical media may be needed for such a transfer: + +```shell +#!/bin/bash +set -ux + +# Specify needed analyzer images +analyzers=${SAST_ANALYZERS:-"bandit eslint gosec"} +registry=$GITLAB_HOST:4567 + +for i in "${analyzers[@]}" +do + tarname="${i}_2.tar" + scp ./analyzers/${tarname} ${GITLAB_HOST}:~/${tarname} + ssh $GITLAB_HOST "sudo docker load -i ${tarname}" + ssh $GITLAB_HOST "sudo docker tag $(sudo docker images | grep $i | awk '{print $3}') ${registry}/analyzers/${i}:2" + ssh $GITLAB_HOST "sudo docker push ${registry}/analyzers/${i}:2" +done +``` diff --git a/doc/topics/airgap/quick_start_guide.md b/doc/topics/offline/quick_start_guide.md index 8d0ff3558ce..0abdd08ffcf 100644 --- a/doc/topics/airgap/quick_start_guide.md +++ b/doc/topics/offline/quick_start_guide.md @@ -74,7 +74,7 @@ Follow these steps to enable the container registry. Note that these steps refle sudo gitlab-ctl reconfigure ``` -## Allow the docker daemon to trust the registry and GitLab Runner +## Allow the Docker daemon to trust the registry and GitLab Runner Provide your Docker daemon with your certs by [following the steps for using trusted certificates with your registry](../../administration/packages/container_registry.md#using-self-signed-certificates-with-container-registry): @@ -125,7 +125,7 @@ Now we must add some additional configuration to our runner: Make the following changes to `/etc/gitlab-runner/config.toml`: -- Add docker socket to volumes `volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"]` +- Add Docker socket to volumes `volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"]` - Add `pull_policy = "if-not-present"` to the executor configuration Now we can start our Runner: diff --git a/doc/topics/web_application_firewall/index.md b/doc/topics/web_application_firewall/index.md index 9f3289cd797..57043bf73b3 100644 --- a/doc/topics/web_application_firewall/index.md +++ b/doc/topics/web_application_firewall/index.md @@ -70,7 +70,7 @@ more advanced rules around threat detection. ## Features -ModSecurity is enabled with the [OWASP Core Rule Set (CRS)](https://modsecurity.org/crs/) by +ModSecurity is enabled with the [OWASP Core Rule Set (CRS)](https://github.com/coreruleset/coreruleset/) by default. The OWASP CRS logs attempts to the following attacks: - [SQL Injection](https://wiki.owasp.org/index.php/OWASP_Periodic_Table_of_Vulnerabilities_-_SQL_Injection) @@ -87,7 +87,7 @@ It is good to have a basic knowledge of the following: - [Kubernetes](https://kubernetes.io/docs/home/) - [Ingress](https://kubernetes.github.io/ingress-nginx/) - [ModSecurity](https://www.modsecurity.org/) -- [OWASP Core Rule Set](https://modsecurity.org/crs/) +- [OWASP Core Rule Set](https://github.com/coreruleset/coreruleset/) ## Roadmap diff --git a/doc/topics/web_application_firewall/quick_start_guide.md b/doc/topics/web_application_firewall/quick_start_guide.md index d55ab03a3f2..ec6702bb457 100644 --- a/doc/topics/web_application_firewall/quick_start_guide.md +++ b/doc/topics/web_application_firewall/quick_start_guide.md @@ -150,7 +150,7 @@ By now you should see the pipeline running, but what is it running exactly? To navigate inside the pipeline, click its status badge (its status should be "Running"). The pipeline is split into a few stages, each running a couple of jobs. - + In the **build** stage, the application is built into a Docker image and then uploaded to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](../autodevops/stages.md#auto-build)). @@ -254,5 +254,5 @@ You can explore them in more detail: - [GitLab Defend Vision](https://about.gitlab.com/direction/defend/#waf) - [ModSecurity](https://www.modsecurity.org/) -- [OWASP Core Rule Set](https://modsecurity.org/crs/) +- [OWASP Core Rule Set](https://github.com/coreruleset/coreruleset/) - [AutoDevOps](../autodevops/index.md) diff --git a/doc/university/README.md b/doc/university/README.md index 84e3b84139b..df41167dbe9 100644 --- a/doc/university/README.md +++ b/doc/university/README.md @@ -11,7 +11,7 @@ If you're looking for a GitLab subscription for _your university_, see our [GitL CAUTION: **Caution:** Some of the content in GitLab University may be out of date and we plan to -[evaluate](https://gitlab.com/gitlab-org/gitlab-foss/issues/41064) it. +[evaluate](https://gitlab.com/gitlab-org/gitlab/-/issues/20403) it. The GitLab University curriculum is composed of GitLab videos, screencasts, presentations, projects and external GitLab content hosted on other services and has been organized into the following sections: @@ -76,7 +76,6 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres - Getting Technical Support - Being part of our Great Community and Contributing to GitLab 1. [Getting Started with the GitLab Development Kit (GDK)](https://about.gitlab.com/blog/2016/06/08/getting-started-with-gitlab-development-kit/) -1. [Contributing Technical Articles to the GitLab Blog](https://about.gitlab.com/blog/2016/01/26/call-for-writers/) 1. [GitLab Training Workshops](training/end-user/README.md) 1. [GitLab Professional Services](https://about.gitlab.com/services/) @@ -176,7 +175,7 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres 1. [Scalability and High Availability - Video](https://www.youtube.com/watch?v=cXRMJJb6sp4&list=PLFGfElNsQthbQu_IWlNOxul0TbS_2JH-e&index=2) 1. [High Availability - Video](https://www.youtube.com/watch?v=36KS808u6bE&index=15&list=PLFGfElNsQthbQu_IWlNOxul0TbS_2JH-e) -1. [High Availability Documentation](https://about.gitlab.com/solutions/high-availability/) +1. [High Availability Documentation](https://about.gitlab.com/solutions/reference-architectures/) ### 3.8 Cycle Analytics @@ -204,7 +203,6 @@ The GitLab University curriculum is composed of GitLab videos, screencasts, pres NOTE: **Note:** Some content can only be accessed by GitLab team members. -1. [Support Path](support/README.md) 1. [Sales Path](https://about.gitlab.com/handbook/sales/onboarding/) 1. [User Training](training/user_training.md) 1. [GitLab Flow Training](training/gitlab_flow.md) diff --git a/doc/university/support/README.md b/doc/university/support/README.md deleted file mode 100644 index 5e824bcb6f9..00000000000 --- a/doc/university/support/README.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -comments: false -type: reference ---- - -# Support Boot Camp - -**Goal:** Prepare new Service Engineers at GitLab - -For each stage, there are learning goals and content to support the learning of the engineer. -The goal of this boot camp is to have every Service Engineer prepared to help our customers -with whatever needs they might have and to also assist our awesome community with their -questions. - -Always start with the [University Overview](../README.md) and then work -your way here for more advanced and specific training. Once you feel comfortable -with the topics of the current stage, move to the next. - -## Stage 1 - -Follow the topics on the [University Overview](../README.md), concentrate on it -during your first Stage, but also: - -- Perform the [first steps](https://about.gitlab.com/handbook/support/onboarding/#first-steps) of - the on-boarding process for new Service Engineers - -### Goals - -Aim to have a good overview of the Product and main features, Git and the Company - -## Stage 2 - -Continue to look over remaining portions of the [University Overview](../README.md) and continue on to these topics: - -### Set up your development machine - -Get your development machine ready to familiarize yourself with the codebase, the components, and to be prepared to reproduce issues that our users encounter - -- Install the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) - - [Set up OpenLDAP as part of this](https://gitlab.com/gitlab-org/gitlab-development-kit#openldap) - -### Become comfortable with the Installation processes that we support - -It's important to understand how to install GitLab in the same way that our users do. Try installing different versions and upgrading and downgrading between them. Installation from source will give you a greater understanding of the components that we employ and how everything fits together. - -Sometimes we need to upgrade customers from old versions of GitLab to latest, so it's good to get some experience of doing that now. - -- [Installation Methods](https://about.gitlab.com/install/): - - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/) - - [Docker](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/docker) - - [Source](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/install/installation.md) -- Get yourself a Digital Ocean droplet, where you can install and maintain your own instance of GitLab - - Ask in #infrastructure about this - - Populate with some test data - - Keep this up-to-date as patch and version releases become available, just like our customers would -- Try out the following installation path - - [Install GitLab 4.2 from source](https://gitlab.com/gitlab-org/gitlab-foss/blob/d67117b5a185cfb15a1d7e749588ff981ffbf779/doc/install/installation.md) - - External MySQL database - - External NGINX - - Create some test data - - Populated Repos - - Users - - Groups - - Projects - - [Back up using our backup Rake task](../../raketasks/backup_restore.md#back-up-gitlab) - - [Upgrade to 5.0 source using our Upgrade documentation](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/4.2-to-5.0.md) - - [Upgrade to 5.1 source](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/5.0-to-5.1.md) - - [Upgrade to 6.0 source](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/5.1-to-6.0.md) - - [Upgrade to 7.14 source](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/6.x-or-7.x-to-7.14.md) - - [Perform the MySQL to PostgreSQL migration to convert your backup](../../update/mysql_to_postgresql.md) - - [Upgrade to Omnibus 7.14](https://docs.gitlab.com/omnibus/update/README.html#upgrading-from-a-non-omnibus-installation-to-an-omnibus-installation) - - [Restore backup using our Restore Rake task](../../raketasks/backup_restore.md#restore-gitlab) - - [Upgrade to latest EE](https://about.gitlab.com/update/) - - (GitLab inc. only) Acquire and apply a license for the Enterprise Edition product, ask in #support -- Perform a downgrade from [EE to CE](../../downgrade_ee_to_ce/README.md) - -### Start to learn about some of the integrations that we support - -Our integrations add great value to GitLab. User questions often relate to integrating GitLab with existing external services and the configuration involved - -- Learn about our Integrations (specially, not only): - - [LDAP](../../integration/ldap.md) - - [Jira](../../project_services/jira.md) - - [Jenkins](../../integration/jenkins.md) - - [SAML](../../integration/saml.md) - -### Goals - -- Aim to be comfortable with installation of the GitLab product and configuration of some of the major integrations -- Aim to have an installation available for reproducing customer reports - -## Stage 3 - -### Understand the gathering of diagnostics for GitLab instances - -- Learn about the GitLab checks that are available: - - [Environment Information and maintenance checks](../../raketasks/maintenance.md) - - [GitLab check](../../raketasks/check.md) - - Omnibus commands - - [Status](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/maintenance/README.md#get-service-status) - - [Starting and stopping services](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/maintenance/README.md#starting-and-stopping) - - [Starting a rails console](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/maintenance/README.md#invoking-rake-tasks) - -### Learn about the Support process - -Zendesk is our Support Center and our main communication line with our Customers. We communicate with customers through several other channels too - -- Familiarize yourself with ZenDesk: - - [UI Overview](https://support.zendesk.com/hc/en-us/articles/203661806-Introduction-to-the-Zendesk-agent-interface) - - [Updating Tickets](https://support.zendesk.com/hc/en-us/articles/212530318-Updating-and-solving-tickets) - - [Working w/ Tickets](https://support.zendesk.com/hc/en-us/articles/203690856-Working-with-tickets) *Read: avoiding agent collision.* -- Dive into our ZenDesk support process by reading how to [handle tickets](https://about.gitlab.com/handbook/support/onboarding/#handling-tickets) -- Start getting real world experience by handling real tickets, all the while gaining further experience with the Product. - - First, learn about our [Support Channels](https://about.gitlab.com/handbook/support/#support-channels) - - Ask other Service Engineers for help, when necessary, and to review your responses - - Start with [StackOverflow](https://about.gitlab.com/handbook/support/#stack-overflowa-namestack-overflowa) and the [GitLab forum](https://about.gitlab.com/handbook/support/#foruma-namegitlab-foruma) - - Here you will find a large variety of queries mainly from our Users who are self hosting GitLab CE - - Understand the questions that are asked and dig in to try to find a solution - - [Proceed on to the GitLab.com Support Forum](https://about.gitlab.com/handbook/support/#gitlabcom-support-trackera-namesupp-foruma) - - Here you will find queries regarding our own GitLab.com - - Helping Users here will give you an understanding of our Admin interface and other tools - - [Proceed on to the Twitter tickets in Zendesk](https://about.gitlab.com/handbook/support/#twitter) - - Here you will gain a great insight into our userbase - - Learn from any complaints and problems and feed them back to the team - - Tweets can range from help needed with GitLab installations, the API and just general queries - - [Proceed on to Regular email Support tickets](https://about.gitlab.com/handbook/support/#regular-zendesk-tickets-a-nameregulara) - - Here you will find tickets from our GitLab EE Customers and GitLab CE Users - - Tickets here are extremely varied and often very technical - - You should be prepared for these tickets, given the knowledge gained from previous tiers and your training -- Check out your colleagues' responses - - Hop on to the #support-live-feed channel in Slack and see the tickets as they come in and are updated - - Read through old tickets that your colleagues have worked on -- Start arranging to pair on calls with other Service Engineers. Aim to cover a few of each type of call - - [Learn about Cisco WebEx](https://about.gitlab.com/handbook/support/onboarding/#webexa-namewebexa) - - Training calls - - Information gathering calls - - It's good to find out how new and prospective customers are going to be using the product and how they will set up their infrastructure - - Diagnosis calls - - When email isn't enough we may need to hop on a call and do some debugging along side the customer - - These paired calls are a great learning experience - - Upgrade calls - - Emergency calls - -### Learn about the Escalation process for tickets - -Some tickets need specific knowledge or a deep understanding of a particular component and will need to be escalated to a Senior Service Engineer or Developer - -- Read about [Escalation](https://about.gitlab.com/handbook/support/workflows/issue_escalations.html) -- Find the macros in Zendesk for ticket escalations -- Take a look at the [GitLab.com Team page](https://about.gitlab.com/company/team/) to find the resident experts in their fields - -### Learn about raising issues and fielding feature proposals - -- Understand what's in the pipeline and proposed features at GitLab: [Direction Page](https://about.gitlab.com/direction/) -- Practice searching issues and filtering using [labels](https://gitlab.com/gitlab-org/gitlab/-/labels) to find existing feature proposals and bugs -- If raising a new issue always provide a relevant label and a link to the relevant ticket in Zendesk -- Add [customer labels](https://gitlab.com/gitlab-org/gitlab-foss/issues?label_name%5B%5D=customer) for those issues relevant to our subscribers -- Take a look at the [existing issue templates](https://gitlab.com/gitlab-org/gitlab/blob/master/CONTRIBUTING.md#issue-tracker) to see what is expected -- Raise issues for bugs in a manner that would make the issue easily reproducible. A Developer or a contributor may work on your issue - -### Goals - -- Aim to have a good understanding of the problems that customers are facing -- Aim to have gained experience in scheduling and participating in calls with customers -- Aim to have a good understanding of ticket flow through Zendesk and how to interact with our various channels - -## Stage 4 - -### Advanced GitLab topics - -Move on to understanding some of GitLab's more advanced features. You can make use of GitLab.com to understand the features from an end-user perspective and then use your own instance to understand setup and configuration of the feature from an Administrative perspective - -- Set up and try [Git LFS](../../topics/git/lfs/index.md) -- Get to know the [GitLab API](../../api/README.md), its capabilities and shortcomings -- Learn how to [migrate from SVN to Git](../../user/project/import/svn.md) -- Set up [GitLab CI/CD](../../ci/quick_start/README.md) -- Create your first [GitLab Page](../../administration/pages/index.md) -- Get to know the GitLab Codebase by reading through the source code: - - Find the differences between the [EE codebase](https://gitlab.com/gitlab-org/gitlab-foss) - and the [CE codebase](https://gitlab.com/gitlab-org/gitlab-foss) -- Ask as many questions as you can think of on the `#support` chat channel - -### Get initiated for on-call duty - -- Read over the [public run-books to understand common tasks](https://gitlab.com/gitlab-com/runbooks) -- Create an issue on the internal Organization tracker to schedule time with the DevOps / Production team, so that you learn how to handle GitLab.com going down. Once you are trained for this, you are ready to be added to the on-call rotation. - -### Goals - -- Aim to become a fully-fledged Service Engineer! - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> diff --git a/doc/university/training/user_training.md b/doc/university/training/user_training.md index 663ae0126c9..d2def1f3199 100644 --- a/doc/university/training/user_training.md +++ b/doc/university/training/user_training.md @@ -222,7 +222,7 @@ See GitLab merge requests for examples: <https://gitlab.com/gitlab-org/gitlab-fo - Dashboard - User Preferences -- ReadMe, Changelog, License shortcuts +- README, Changelog, License shortcuts - Issues - Milestones and Labels - Manage project members @@ -314,7 +314,7 @@ Squash the commits on the same branch we used for the merge conflicts step. git rebase -i master ``` -In the editor, leave the first commit as 'pick' and set others to 'fixup'. +In the editor, leave the first commit as `pick` and set others to `fixup`. ## Questions? diff --git a/doc/update/README.md b/doc/update/README.md index 8056460ceaa..f36a304495c 100644 --- a/doc/update/README.md +++ b/doc/update/README.md @@ -22,7 +22,7 @@ Based on your installation, choose a section below that fits your needs. source](upgrading_from_source.md) - The guidelines for upgrading Community Edition and Enterprise Edition from source. - [Patch versions](patch_versions.md) guide includes the steps needed for a - patch version, eg. 6.2.0 to 6.2.1, and apply to both Community and Enterprise + patch version, such as 6.2.0 to 6.2.1, and apply to both Community and Enterprise Editions. In the past we used separate documents for the upgrading instructions, but we @@ -178,7 +178,7 @@ If you wish to upgrade your GitLab installation from Community to Enterprise Edition, follow the guides below based on the installation method: - [Source CE to EE update guides](upgrading_from_ce_to_ee.md) - The steps are very similar - to a version upgrade: stop the server, get the code, update config files for + to a version upgrade: stop the server, get the code, update configuration files for the new functionality, install libraries and do migrations, update the init script, start the application and check its status. - [Omnibus CE to EE](https://docs.gitlab.com/omnibus/update/README.html#updating-community-edition-to-enterprise-edition) - Follow this guide to update your Omnibus @@ -192,6 +192,16 @@ possible. ## Version specific upgrading instructions +### 13.1.0 + +In 13.1.0, you must upgrade to either: + +- At least Git v2.24 (previously, the minimum required version was Git v2.22). +- The recommended Git v2.26. + +Failure to do so will result in internal errors in the Gitaly service in some RPCs due +to the use of the new `--end-of-options` Git flag. + ### 12.2.0 In 12.2.0, we enabled Rails' authenticated cookie encryption. Old sessions are diff --git a/doc/update/patch_versions.md b/doc/update/patch_versions.md index f226ad9ac28..0092df7b89d 100644 --- a/doc/update/patch_versions.md +++ b/doc/update/patch_versions.md @@ -7,7 +7,7 @@ comments: false ## Select Version to Install Make sure you view [this update guide](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/update/patch_versions.md) from the tag (version) of GitLab you would like to install. -In most cases this should be the highest numbered production tag (without rc in it). +In most cases this should be the highest numbered production tag (without `rc` in it). You can select the tag in the version dropdown in the top left corner of GitLab (below the menu bar). ### 0. Backup @@ -35,7 +35,7 @@ sudo -u git -H git checkout -- Gemfile.lock db/structure.sql locale sudo -u git -H git checkout LATEST_TAG -b LATEST_TAG ``` -### 3. Install libs, migrations, etc +### 3. Install libraries, migrations, etc ```shell cd /home/git/gitlab @@ -49,10 +49,9 @@ sudo -u git -H bundle clean sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production # Compile GetText PO files -# Internationalization was added in `v9.2.0` so these commands are only +# Internationalization was added in `v9.2.0` so this command is only # required for versions equal or major to it. -sudo -u git -H bundle exec rake gettext:pack RAILS_ENV=production -sudo -u git -H bundle exec rake gettext:po_to_json RAILS_ENV=production +sudo -u git -H bundle exec rake gettext:compile RAILS_ENV=production # Clean up assets and cache sudo -u git -H bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production NODE_ENV=production NODE_OPTIONS="--max_old_space_size=4096" diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md index 2415d3b2741..78227f18457 100644 --- a/doc/update/upgrading_from_ce_to_ee.md +++ b/doc/update/upgrading_from_ce_to_ee.md @@ -54,7 +54,7 @@ sudo -u git -H git remote add -f ee https://gitlab.com/gitlab-org/gitlab.git sudo -u git -H git checkout EE_BRANCH ``` -### 3. Install libs, migrations, etc +### 3. Install libraries, migrations, etc ```shell cd /home/git/gitlab diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index 76a0d5bde98..af461b93910 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -12,7 +12,7 @@ Make sure you view this update guide from the branch (version) of GitLab you would like to install (e.g., `11.8`. You can select the version in the version dropdown at the top left corner of GitLab (below the menu bar). -In all examples, replace `BRANCH` with the branch for the version you uprading +In all examples, replace `BRANCH` with the branch for the version you upgrading to (e.g. `11-8-stable` for `11.8`), and replace `PREVIOUS_BRANCH` with the branch for the version you are upgrading from (e.g. `11-7-stable` for `11.7`). @@ -67,9 +67,9 @@ Download Ruby and compile it: ```shell mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.gz -echo '1416ce288fb8bfeae07a12b608540318c9cace71 ruby-2.6.5.tar.gz' | shasum -c - && tar xzf ruby-2.6.5.tar.gz -cd ruby-2.6.5 +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.6.tar.gz +echo '2d78048e293817f38d4ede4ebc7873013e97bb0b ruby-2.6.6.tar.gz' | shasum -c - && tar xzf ruby-2.6.6.tar.gz +cd ruby-2.6.6 ./configure --disable-install-rdoc make @@ -122,12 +122,17 @@ rm go1.13.5.linux-amd64.tar.gz ### 6. Update Git -NOTE: To check the minimum required Git version, see [Git versions](../install/requirements.md#git-versions). +CAUTION: **Caution:** +From GitLab 13.1, you must use at least Git v2.24 (previous minimum version was v2.22). +Git v2.26 is recommended. + +To check you are running the minimum required Git version, see +[Git versions](../install/requirements.md#git-versions). In Debian or Ubuntu: ```shell -# Make sure Git is version 2.21.0 or higher +# Make sure Git is version 2.24.0 or higher git --version # Remove packaged Git @@ -147,9 +152,9 @@ make install # Download and compile from source cd /tmp -curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.21.0.tar.gz -echo '85eca51c7404da75e353eba587f87fea9481ba41e162206a6f70ad8118147bee git-2.21.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.21.0.tar.gz -cd git-2.21.0/ +curl --remote-name --location --progress https://www.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz +echo 'aa168c2318e7187cd295a645f7370cc6d71a324aafc932f80f00c780b6a26bed git-2.26.0.tar.gz' | shasum -a256 -c - && tar -xzf git-2.26.0.tar.gz +cd git-2.26.0/ ./configure --with-libpcre make prefix=/usr/local all @@ -285,7 +290,7 @@ add the following line to `config/initializers/smtp_settings.rb`: ActionMailer::Base.delivery_method = :smtp ``` -See [smtp_settings.rb.sample](https://gitlab.com/gitlab-org/gitlab/blob/master/config/initializers/smtp_settings.rb.sample#L13) as an example. +See [`smtp_settings.rb.sample`](https://gitlab.com/gitlab-org/gitlab/blob/master/config/initializers/smtp_settings.rb.sample#L13) as an example. #### Init script @@ -313,7 +318,7 @@ For Ubuntu 16.04.1 LTS: sudo systemctl daemon-reload ``` -### 13. Install libs, migrations, etc +### 13. Install libraries, migrations, etc ```shell cd /home/git/gitlab @@ -378,6 +383,18 @@ Example: Additional instructions here. --> +### 13.0.1 + +As part of [deprecating Rack Attack throttles on Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4750), Rack Attack initializer on GitLab +was renamed from [`config/initializers/rack_attack_new.rb` to `config/initializers/rack_attack.rb`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33072). +If this file exists on your installation, consider creating a backup before updating: + +```shell +cd /home/git/gitlab + +cp config/initializers/rack_attack.rb config/initializers/rack_attack_backup.rb +``` + ## Troubleshooting ### 1. Revert the code to the previous version diff --git a/doc/update/upgrading_postgresql_using_slony.md b/doc/update/upgrading_postgresql_using_slony.md index fdd035c7940..5133e4cf4e5 100644 --- a/doc/update/upgrading_postgresql_using_slony.md +++ b/doc/update/upgrading_postgresql_using_slony.md @@ -56,7 +56,7 @@ server. ## Installing Slony -Slony will be used to upgrade the database without requiring long downtimes. +Slony will be used to upgrade the database without requiring a long downtime. Slony can be downloaded from <https://www.slony.info/>. If you have installed PostgreSQL using your operating system's package manager you may also be able to install Slony using said package manager. @@ -88,7 +88,7 @@ test -f /opt/gitlab/embedded/bin/slonik_init_cluster && echo 'Slony Perl tools a ``` This assumes Slony was installed to `/opt/gitlab/embedded`. If Slony was -installed properly the output of these commands will be (the mentioned "slonik" +installed properly the output of these commands will be (the mentioned `slonik` version may be different): ```plaintext diff --git a/doc/user/admin_area/abuse_reports.md b/doc/user/admin_area/abuse_reports.md index 77f4a84cf6f..783aadc0974 100644 --- a/doc/user/admin_area/abuse_reports.md +++ b/doc/user/admin_area/abuse_reports.md @@ -1,4 +1,7 @@ --- +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- diff --git a/doc/user/admin_area/activating_deactivating_users.md b/doc/user/admin_area/activating_deactivating_users.md index 7c8ab18ccef..06a26737495 100644 --- a/doc/user/admin_area/activating_deactivating_users.md +++ b/doc/user/admin_area/activating_deactivating_users.md @@ -1,4 +1,7 @@ --- +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -8,7 +11,7 @@ GitLab administrators can deactivate and activate users. ## Deactivating a user -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/22257) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22257) in GitLab 12.4. In order to temporarily prevent access by a GitLab user that has no recent activity, administrators can choose to deactivate the user. @@ -45,7 +48,7 @@ A deactivated user does not consume a [seat](../../subscriptions/index.md#choosi ## Activating a user -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/22257) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22257) in GitLab 12.4. A deactivated user can be activated from the Admin Area. @@ -63,4 +66,4 @@ Activating a user will change the user's state to active and it consumes a [seat](../../subscriptions/index.md#choosing-the-number-of-users). TIP: **Tip:** -A deactivated user can also activate their account by themselves by simply logging back via the UI. +A deactivated user can also activate their account themselves by simply logging back in via the UI. diff --git a/doc/user/admin_area/appearance.md b/doc/user/admin_area/appearance.md index 80440b63f71..55dabce7342 100644 --- a/doc/user/admin_area/appearance.md +++ b/doc/user/admin_area/appearance.md @@ -15,7 +15,7 @@ By default, the navigation bar has the GitLab logo, but this can be customized w any image desired. It is optimized for images 28px high (any width), but any image can be used (less than 1MB) and it will automatically be resized. - + Once you select and upload an image, click **Update appearance settings** at the bottom of the page to activate it in the GitLab instance. @@ -38,8 +38,8 @@ of the page to activate it in the GitLab instance. ## System header and footer messages -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5023) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.7. -> - [Added](https://gitlab.com/gitlab-org/gitlab-foss/issues/55057) to [GitLab Core](https://about.gitlab.com/pricing/) in 11.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5023) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.7. +> - [Added](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55057) to [GitLab Core](https://about.gitlab.com/pricing/) in 11.9. You can add a small header message, a small footer message, or both, to the interface of your GitLab instance. These messages will appear on all projects and pages of the diff --git a/doc/user/admin_area/blocking_unblocking_users.md b/doc/user/admin_area/blocking_unblocking_users.md index e3b9cd1218c..2f98709a089 100644 --- a/doc/user/admin_area/blocking_unblocking_users.md +++ b/doc/user/admin_area/blocking_unblocking_users.md @@ -1,4 +1,7 @@ --- +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- diff --git a/doc/user/admin_area/broadcast_messages.md b/doc/user/admin_area/broadcast_messages.md index 5427b0c5200..7e1a4faee75 100644 --- a/doc/user/admin_area/broadcast_messages.md +++ b/doc/user/admin_area/broadcast_messages.md @@ -24,7 +24,7 @@ Banners are shown on the top of a page and in Git remote responses.  -```bash +```shell $ git push ... remote: diff --git a/doc/user/admin_area/credentials_inventory.md b/doc/user/admin_area/credentials_inventory.md index 79e9cbc5b14..5eecfbb73c6 100644 --- a/doc/user/admin_area/credentials_inventory.md +++ b/doc/user/admin_area/credentials_inventory.md @@ -1,3 +1,10 @@ +--- +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Credentials inventory **(ULTIMATE ONLY)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20912) in GitLab 12.6. diff --git a/doc/user/admin_area/custom_project_templates.md b/doc/user/admin_area/custom_project_templates.md index f9a4dad2500..d194ca42215 100644 --- a/doc/user/admin_area/custom_project_templates.md +++ b/doc/user/admin_area/custom_project_templates.md @@ -1,10 +1,13 @@ --- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- # Custom instance-level project templates **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6860) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6860) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. GitLab administrators can configure the group where all the custom project templates are sourced. diff --git a/doc/user/admin_area/diff_limits.md b/doc/user/admin_area/diff_limits.md index bc6f93891df..b9bd94e65be 100644 --- a/doc/user/admin_area/diff_limits.md +++ b/doc/user/admin_area/diff_limits.md @@ -1,4 +1,7 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/user/admin_area/geo_nodes.md b/doc/user/admin_area/geo_nodes.md index bb4ca8ca618..d17e0f7430c 100644 --- a/doc/user/admin_area/geo_nodes.md +++ b/doc/user/admin_area/geo_nodes.md @@ -1,4 +1,7 @@ --- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- diff --git a/doc/user/admin_area/labels.md b/doc/user/admin_area/labels.md index 1d15be89bd5..3c35276b843 100644 --- a/doc/user/admin_area/labels.md +++ b/doc/user/admin_area/labels.md @@ -1,4 +1,7 @@ --- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- diff --git a/doc/user/admin_area/license.md b/doc/user/admin_area/license.md index 59b71b05b16..1ffaf4e0678 100644 --- a/doc/user/admin_area/license.md +++ b/doc/user/admin_area/license.md @@ -1,4 +1,7 @@ --- +stage: Growth +group: Conversion +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- @@ -64,12 +67,12 @@ export GITLAB_LICENSE_FILE="/path/to/license/file" Omnibus installations should add this entry to `gitlab.rb`: ```ruby -gitlab_rails['license_file'] = "/path/to/license/file" +gitlab_rails['initial_license_file'] = "/path/to/license/file" ``` CAUTION: **Caution:** These methods will only add a license at the time of installation. Use the -Admin Area in the web ui to renew or upgrade licenses. +Admin Area in the web user interface to renew or upgrade licenses. --- diff --git a/doc/user/admin_area/merge_requests_approvals.md b/doc/user/admin_area/merge_requests_approvals.md index 0c7beadad48..153ccfc128a 100644 --- a/doc/user/admin_area/merge_requests_approvals.md +++ b/doc/user/admin_area/merge_requests_approvals.md @@ -1,10 +1,13 @@ --- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, concepts --- # Instance-level merge request approval rules **(PREMIUM ONLY)** -> Introduced in [GitLab Premium](https://gitlab.com/gitlab-org/gitlab/issues/39060) 12.8. +> Introduced in [GitLab Premium](https://gitlab.com/gitlab-org/gitlab/-/issues/39060) 12.8. Merge request approvals rules prevent users overriding certain settings on a project level. When configured, only administrators can change these settings on a project level diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 83b29597bec..b4fb7a524da 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -15,6 +15,17 @@ If you choose a size larger than what is currently configured for the web server you will likely get errors. See the [troubleshooting section](#troubleshooting) for more details. +## Max import size + +You can change the maximum file size for imports in GitLab. +Navigate to **Admin Area (wrench icon) > Settings > General**, then expand **Account and Limit**. +From here, you can increase or decrease by changing the value in `Maximum import size (MB)`. + +NOTE: **Note:** +If you choose a size larger than what is currently configured for the web server, +you will likely get errors. See the [troubleshooting section](#troubleshooting) for more +details. + ## Maximum namespace storage size This sets a maximum size limit on each namespace. The following are included in the namespace size: @@ -99,7 +110,7 @@ nginx['client_max_body_size'] = "200m" ## Limiting lifetime of personal access tokens **(ULTIMATE ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3649) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. Users can optionally specify an expiration date for [personal access tokens](../../profile/personal_access_tokens.md). @@ -131,7 +142,7 @@ Once a lifetime for personal access tokens is set, GitLab will: ## Disabling user profile name changes **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/24605) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24605) in GitLab 12.7. To maintain integrity of user details in [Audit Events](../../../administration/audit_events.md), GitLab administrators can choose to disable a user's ability to change their profile name. diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index 4a401210928..3a287f29a0a 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -1,4 +1,7 @@ --- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference --- @@ -32,7 +35,7 @@ The maximum size of the [job artifacts](../../../administration/job_artifacts.md can be set at: - The instance level. -- [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/21688), the project and group level. +- [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/21688), the project and group level. The value is: @@ -169,7 +172,7 @@ but commented out to help encourage others to add to it in the future. --> CAUTION: **Caution:** This feature is being re-evaluated in favor of a different -[compliance solution](https://gitlab.com/gitlab-org/gitlab/issues/34830). +[compliance solution](https://gitlab.com/gitlab-org/gitlab/-/issues/34830). We recommend that users who haven't yet implemented this feature wait for the new solution. diff --git a/doc/user/admin_area/settings/email.md b/doc/user/admin_area/settings/email.md index a99897657ae..d956364b3ea 100644 --- a/doc/user/admin_area/settings/email.md +++ b/doc/user/admin_area/settings/email.md @@ -1,7 +1,11 @@ --- type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- + # Email **(CORE ONLY)** You can customize some of the content in emails sent from your GitLab instance. diff --git a/doc/user/admin_area/settings/external_authorization.md b/doc/user/admin_area/settings/external_authorization.md index 3a03e46fa1f..c5c5f08aea1 100644 --- a/doc/user/admin_area/settings/external_authorization.md +++ b/doc/user/admin_area/settings/external_authorization.md @@ -4,7 +4,7 @@ type: reference # External authorization control **(CORE ONLY)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4216) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4216) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.6. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/27056) to [GitLab Core](https://about.gitlab.com/pricing/) in 11.10. In highly controlled environments, it may be necessary for access policy to be @@ -31,7 +31,7 @@ functionality that render cross-project data. That includes: This is to prevent performing to many requests at once to the external authorization service. -Whenever access is granted or denied this is logged in a logfile called +Whenever access is granted or denied this is logged in a log file called `external-policy-access-control.log`. Read more about logs GitLab keeps in the [omnibus documentation](https://docs.gitlab.com/omnibus/settings/logs.html). @@ -60,7 +60,7 @@ The available required properties are: requesting authorization if no specific label is defined on the project When using TLS Authentication with a self signed certificate, the CA certificate -needs to be trusted by the openssl installation. When using GitLab installed using +needs to be trusted by the OpenSSL installation. When using GitLab installed using Omnibus, learn to install a custom CA in the [omnibus documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). Alternatively learn where to install custom certificates using `openssl version -d`. diff --git a/doc/user/admin_area/settings/gitaly_timeouts.md b/doc/user/admin_area/settings/gitaly_timeouts.md index 775a99d7574..68f359368f0 100644 --- a/doc/user/admin_area/settings/gitaly_timeouts.md +++ b/doc/user/admin_area/settings/gitaly_timeouts.md @@ -4,7 +4,7 @@ type: reference # Gitaly timeouts - + 3 timeout types can be configured to make sure that long running Gitaly calls don't needlessly take up resources. diff --git a/doc/user/admin_area/settings/img/rate_limit_on_issues_creation.png b/doc/user/admin_area/settings/img/rate_limit_on_issues_creation.png Binary files differdeleted file mode 100644 index 5aa9b95f835..00000000000 --- a/doc/user/admin_area/settings/img/rate_limit_on_issues_creation.png +++ /dev/null diff --git a/doc/user/admin_area/settings/img/rate_limit_on_issues_creation_v13_1.png b/doc/user/admin_area/settings/img/rate_limit_on_issues_creation_v13_1.png Binary files differnew file mode 100644 index 00000000000..9edc916c7fb --- /dev/null +++ b/doc/user/admin_area/settings/img/rate_limit_on_issues_creation_v13_1.png diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index 761d640c535..df087722fcf 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -32,7 +32,7 @@ Access the default page for admin area settings by navigating to | Option | Description | | ------ | ----------- | | [Elasticsearch](../../../integration/elasticsearch.md#enabling-elasticsearch) | Elasticsearch integration. Elasticsearch AWS IAM. | -| [PlantUML](../../../administration/integration/plantuml.md#gitlab) | Allow rendering of PlantUML diagrams in Asciidoc documents. | +| [PlantUML](../../../administration/integration/plantuml.md#gitlab) | Allow rendering of PlantUML diagrams in AsciiDoc documents. | | [Slack application](../../../user/project/integrations/gitlab_slack_application.md#configuration) **(FREE ONLY)** | Slack integration allows you to interact with GitLab via slash commands in a chat window. This option is only available on GitLab.com, though it may be [available for self-managed instances in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). | | [Third party offers](third_party_offers.md) | Control the display of third party offers. | | [Snowplow](../../../development/telemetry/snowplow.md) | Configure the Snowplow integration. | diff --git a/doc/user/admin_area/settings/instance_template_repository.md b/doc/user/admin_area/settings/instance_template_repository.md index bef4e31259f..ae56c67f0ab 100644 --- a/doc/user/admin_area/settings/instance_template_repository.md +++ b/doc/user/admin_area/settings/instance_template_repository.md @@ -4,7 +4,7 @@ type: reference # Instance template repository **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5986) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5986) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. ## Overview diff --git a/doc/user/admin_area/settings/push_event_activities_limit.md b/doc/user/admin_area/settings/push_event_activities_limit.md index 9850de0f4b3..bbc5ed04202 100644 --- a/doc/user/admin_area/settings/push_event_activities_limit.md +++ b/doc/user/admin_area/settings/push_event_activities_limit.md @@ -4,7 +4,7 @@ type: reference # Push event activities limit and bulk push events -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31007) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31007) in GitLab 12.4. This allows you to set the number of changes (branches or tags) in a single push to determine whether individual push events or bulk push event will be created. diff --git a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md index dc23865e730..bae60aba15f 100644 --- a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md @@ -1,5 +1,8 @@ --- type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Rate limits on issue creation @@ -14,7 +17,7 @@ For example, requests using the [Projects::IssuesController#create](https://gitlab.com/gitlab-org/gitlab/raw/master/app/controllers/projects/issues_controller.rb) action exceeding a rate of 300 per minute are blocked. Access to the endpoint is allowed after one minute. - + This limit is: diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index 95748f68a55..d9ca4a0881a 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -66,7 +66,7 @@ To ensure only admin users can delete projects: ## Default deletion adjourned period **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32935) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. By default, a project or group marked for removal will be permanently removed after 7 days. This period may be changed, and setting this period to 0 will enable immediate removal diff --git a/doc/user/analytics/code_review_analytics.md b/doc/user/analytics/code_review_analytics.md index e0aa01c29b2..8c4c54153bb 100644 --- a/doc/user/analytics/code_review_analytics.md +++ b/doc/user/analytics/code_review_analytics.md @@ -2,14 +2,13 @@ description: "Learn how long your open merge requests have spent in code review, and what distinguishes the longest-running." # Up to ~200 chars long. They will be displayed in Google Search snippets. It may help to write the page intro first, and then reuse it here. stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Code Review Analytics **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/38062) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38062) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.7. Code Review Analytics makes it easy to view the longest-running reviews among open merge requests, enabling you to take action on individual merge requests and reduce overall cycle time. diff --git a/doc/user/analytics/img/vsa_time_metrics_v13_0.png b/doc/user/analytics/img/vsa_time_metrics_v13_0.png Binary files differnew file mode 100644 index 00000000000..073976fd740 --- /dev/null +++ b/doc/user/analytics/img/vsa_time_metrics_v13_0.png diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index 48df7bc340a..18f6d79ef23 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -1,15 +1,14 @@ --- stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Analytics ## Analytics workspace -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12077) in GitLab 12.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12077) in GitLab 12.2. The Analytics workspace will make it possible to aggregate analytics across GitLab, so that users can view information across multiple projects and groups @@ -19,7 +18,7 @@ To access the Analytics workspace, click on **More > Analytics** in the top navi ## Group-level analytics -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/195979) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195979) in GitLab 12.8. The following analytics features are available at the group level: diff --git a/doc/user/analytics/productivity_analytics.md b/doc/user/analytics/productivity_analytics.md index d0fda61d6a5..653836de8be 100644 --- a/doc/user/analytics/productivity_analytics.md +++ b/doc/user/analytics/productivity_analytics.md @@ -1,17 +1,16 @@ --- stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Productivity Analytics **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12079) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12079) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. Track development velocity with Productivity Analytics. -For many companies, the development cycle is a blackbox and getting an estimate of how +For many companies, the development cycle is a black box and getting an estimate of how long, on average, it takes to deliver features is an enormous endeavor. While [Value Stream Analytics](../project/cycle_analytics.md) focuses on the entire @@ -49,7 +48,7 @@ The following metrics and visualizations are available on a project or group lev ## Date ranges -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13188) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13188) in GitLab 12.4. GitLab has the ability to filter analytics based on a date range. To filter results: diff --git a/doc/user/analytics/repository_analytics.md b/doc/user/analytics/repository_analytics.md index 17032990b09..6d2de951a55 100644 --- a/doc/user/analytics/repository_analytics.md +++ b/doc/user/analytics/repository_analytics.md @@ -1,8 +1,7 @@ --- stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Repository Analytics @@ -34,6 +33,7 @@ The data in the charts are updated soon after each commit in the default branch. Available charts: - Programming languages used in the repository +- Code coverage history (last 3 months) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33743) in GitLab 13.1) - Commit statistics (last month) - Commits per day of month - Commits per weekday diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index 90a4f96f00b..0efe28ac5f7 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -1,14 +1,13 @@ --- stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Value Stream Analytics > - Introduced as Cycle Analytics prior to GitLab 12.3 at the project level. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12077) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3 at the group level. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12077) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3 at the group level. > - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23427) from Cycle Analytics to Value Stream Analytics in GitLab 12.8. Value Stream Analytics measures the time spent to go from an @@ -47,11 +46,11 @@ There are seven stages that are tracked as part of the Value Stream Analytics ca - **Staging** (Continuous Deployment) - Time between merging and deploying to production - **Total** (Total) - - Total lifecycle time. That is, the velocity of the project or team. [Previously known](https://gitlab.com/gitlab-org/gitlab/issues/38317) as **Production**. + - Total lifecycle time. That is, the velocity of the project or team. [Previously known](https://gitlab.com/gitlab-org/gitlab/-/issues/38317) as **Production**. ## Date ranges -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13216) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13216) in GitLab 12.4. GitLab provides the ability to filter analytics based on a date range. To filter results: @@ -59,9 +58,20 @@ GitLab provides the ability to filter analytics based on a date range. To filter 1. Optionally select a project. 1. Select a date range using the available date pickers. -## How the data is measured +## How Time metrics are measured -Value Stream Analytics records cycle time and data based on the project issues with the +The "Time" metrics near the top of the page are measured as follows: + +- **Lead time**: median time from issue created to issue closed. +- **Cycle time**: median time from first commit to issue closed. + +Note: A commit is associated with an issue by [crosslinking](../project/issues/crosslinking_issues.md) in the commit message or by manually linking the merge request containing the commit. + + + +## How the stages are measured + +Value Stream Analytics records stage time and data based on the project issues with the exception of the staging and total stages, where only data deployed to production are measured. @@ -79,7 +89,7 @@ Each stage of Value Stream Analytics is further described in the table below. | Test | Measures the median time to run the entire pipeline for that project. It's related to the time GitLab CI/CD takes to run every job for the commits pushed to that merge request defined in the previous stage. It is basically the start->finish time for all pipelines. | | Review | Measures the median time taken to review the merge request that has a closing issue pattern, between its creation and until it's merged. | | Staging | Measures the median time between merging the merge request with a closing issue pattern until the very first deployment to production. It's tracked by the environment set to `production` or matching `production/*` (case-sensitive, `Production` won't work) in your GitLab CI/CD configuration. If there isn't a production environment, this is not tracked. | -| Total | The sum of all time (medians) taken to run the entire process, from issue creation to deploying the code to production. [Previously known](https://gitlab.com/gitlab-org/gitlab/issues/38317) as **Production**. | +| Total | The sum of all time (medians) taken to run the entire process, from issue creation to deploying the code to production. [Previously known](https://gitlab.com/gitlab-org/gitlab/-/issues/38317) as **Production**. | How this works, behind the scenes: @@ -274,7 +284,7 @@ from within the chart itself. ### Chart median line -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36675) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36675) in GitLab 12.7. The median line on the chart displays data that is offset by the number of days selected. For example, if 30 days worth of data has been selected (for example, 2019-12-16 to 2020-01-15) the diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index dfddbde379d..f0fcd0c4419 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Secure +group: Static Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Security Configuration **(ULTIMATE)** diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 6e52d7dbdcf..0ffe83cdfc9 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Defend +group: Container Security +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Container Scanning **(ULTIMATE)** @@ -8,35 +11,28 @@ type: reference, howto ## Overview -If you are using [GitLab CI/CD](../../../ci/README.md), you can check your Docker -images (or more precisely the containers) for known vulnerabilities by using -[Clair](https://github.com/quay/clair) and [klar](https://github.com/optiopay/klar), -two open source tools for Vulnerability Static Analysis for containers. +Your application's Docker image may itself be based on Docker images that contain known +vulnerabilities. By including an extra job in your pipeline that scans for those vulnerabilities and +displays them in a merge request, you can use GitLab to audit your Docker-based apps. +By default, container scanning in GitLab is based on [Clair](https://github.com/quay/clair) and +[Klar](https://github.com/optiopay/klar), which are open-source tools for vulnerability static analysis in +containers. [GitLab's Klar analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/klar/) +scans the containers and serves as a wrapper for Clair. -You can take advantage of Container Scanning by either [including the CI job](#configuration) in -your existing `.gitlab-ci.yml` file or by implicitly using -[Auto Container Scanning](../../../topics/autodevops/stages.md#auto-container-scanning-ultimate) -that is provided by [Auto DevOps](../../../topics/autodevops/index.md). - -GitLab checks the Container Scanning report, compares the found vulnerabilities -between the source and target branches, and shows the information right on the -merge request. - - - -## Contribute your scanner +NOTE: **Note:** +To integrate security scanners other than Clair and Klar into GitLab, see +[Security scanner integration](../../../development/integrations/secure.md). -The [Security Scanner Integration](../../../development/integrations/secure.md) documentation explains how to integrate other security scanners into GitLab. +You can enable container scanning by doing one of the following: -## Use cases +- [Include the CI job](#configuration) in your existing `.gitlab-ci.yml` file. +- Implicitly use [Auto Container Scanning](../../../topics/autodevops/stages.md#auto-container-scanning-ultimate) + provided by [Auto DevOps](../../../topics/autodevops/index.md). -If you distribute your application with Docker, then there's a great chance -that your image is based on other Docker images that may in turn contain some -known vulnerabilities that could be exploited. +GitLab compares the found vulnerabilities between the source and target branches, and shows the +information directly in the merge request. -Having an extra job in your pipeline that checks for those vulnerabilities, -and the fact that they are displayed inside a merge request, makes it very easy -to perform audits for your Docker-based apps. + <!-- NOTE: The container scanning tool references the following heading in the code, so if you make a change to this heading, make sure to update the documentation URLs used in the @@ -44,33 +40,28 @@ to perform audits for your Docker-based apps. ## Requirements -To enable Container Scanning in your pipeline, you need: - -- A GitLab Runner with the - [`docker`](https://docs.gitlab.com/runner/executors/docker.html) or - [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) - executor. -- Docker `18.09.03` or higher installed on the machine where the Runners are - running. If you're using the shared Runners on GitLab.com, this is already - the case. -- To [build and push](../../packages/container_registry/index.md#container-registry-examples-with-gitlab-cicd) - your Docker image to your project's Container Registry. - The name of the Docker image should use the following - [predefined environment variables](../../../ci/variables/predefined_variables.md) - as defined below: +To enable Container Scanning in your pipeline, you need the following: + +- [GitLab Runner](https://docs.gitlab.com/runner/) with the [Docker](https://docs.gitlab.com/runner/executors/docker.html) + or [Kubernetes](https://docs.gitlab.com/runner/install/kubernetes.html) executor. +- Docker `18.09.03` or higher installed on the same computer as the Runner. If you're using the + shared Runners on GitLab.com, then this is already the case. +- [Build and push](../../packages/container_registry/index.md#container-registry-examples-with-gitlab-cicd) + your Docker image to your project's container registry. The name of the Docker image should use + the following [predefined environment variables](../../../ci/variables/predefined_variables.md): ```plaintext $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA ``` - These can be used directly in your `.gitlab-ci.yml` file: + You can use these directly in your `.gitlab-ci.yml` file: ```yaml build: - image: docker:19.03.8 + image: docker:19.03.11 stage: build services: - - docker:19.03.8-dind + - docker:19.03.11-dind variables: IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA script: @@ -81,45 +72,40 @@ To enable Container Scanning in your pipeline, you need: ## Configuration -For GitLab 11.9 and later, to enable Container Scanning, you must -[include](../../../ci/yaml/README.md#includetemplate) the -[`Container-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml) -that's provided as a part of your GitLab installation. -For GitLab versions earlier than 11.9, you can copy and use the job as defined -in that template. +How you enable Container Scanning depends on your GitLab version: -Add the following to your `.gitlab-ci.yml` file: +- GitLab 11.9 and later: [Include](../../../ci/yaml/README.md#includetemplate) the + [`Container-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml) + that comes with your GitLab installation. +- GitLab versions earlier than 11.9: Copy and use the job from the + [`Container-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml). + +To include the `Container-Scanning.gitlab-ci.yml` template (GitLab 11.9 and later), add the +following to your `.gitlab-ci.yml` file: ```yaml include: - template: Container-Scanning.gitlab-ci.yml ``` -The included template will: +The included template: -1. Create a `container_scanning` job in your CI/CD pipeline. -1. Pull the already built Docker image from your project's - [Container Registry](../../packages/container_registry/index.md) (see [requirements](#requirements)) - and scan it for possible vulnerabilities. +- Creates a `container_scanning` job in your CI/CD pipeline. +- Pulls the built Docker image from your project's [Container Registry](../../packages/container_registry/index.md) + (see [requirements](#requirements)) and scans it for possible vulnerabilities. -The results will be saved as a +GitLab saves the results as a [Container Scanning report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscontainer_scanning-ultimate) -that you can later download and analyze. -Due to implementation limitations, we always take the latest Container Scanning -artifact available. Behind the scenes, the -[GitLab Klar analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/klar/) -is used and runs the scans. +that you can download and analyze later. When downloading, you always receive the most-recent +artifact. -The following is a sample `.gitlab-ci.yml` that will build your Docker image, -push it to the Container Registry, and run Container Scanning: +The following is a sample `.gitlab-ci.yml` that builds your Docker image, pushes it to the Container +Registry, and scans the containers: ```yaml variables: DOCKER_DRIVER: overlay2 -services: - - docker:19.03.8-dind - stages: - build - test @@ -127,6 +113,8 @@ stages: build: image: docker:stable stage: build + services: + - docker:19.03.11-dind variables: IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA script: @@ -141,11 +129,15 @@ include: ### Customizing the Container Scanning settings -You can change container scanning settings by using the [`variables`](../../../ci/yaml/README.md#variables) -parameter in your `.gitlab-ci.yml` to change [environment variables](#available-variables). +There may be cases where you want to customize how GitLab scans your containers. For example, you +may want to enable more verbose output from Clair or Klar, access a Docker registry that requires +authentication, and more. To change such settings, use the [`variables`](../../../ci/yaml/README.md#variables) +parameter in your `.gitlab-ci.yml` to set [environment variables](#available-variables). +The environment variables you set in your `.gitlab-ci.yml` overwrite those in +`Container-Scanning.gitlab-ci.yml`. -In the following example, we [include](../../../ci/yaml/README.md#include) the template and also -set the `CLAIR_OUTPUT` variable to `High`: +This example [includes](../../../ci/yaml/README.md#include) the Container Scanning template and +enables verbose output from Clair by setting the `CLAIR_OUTPUT` environment variable to `High`: ```yaml include: @@ -155,9 +147,6 @@ variables: CLAIR_OUTPUT: High ``` -The `CLAIR_OUTPUT` variable defined in the main `gitlab-ci.yml` will overwrite what's -defined in `Container-Scanning.gitlab-ci.yml`, changing the Container Scanning behavior. - <!-- NOTE: The container scanning tool references the following heading in the code, so if you" make a change to this heading, make sure to update the documentation URLs used in the" container scanning tool (https://gitlab.com/gitlab-org/security-products/analyzers/klar)" --> @@ -188,13 +177,9 @@ using environment variables. ### Overriding the Container Scanning template -CAUTION: **Deprecation:** -Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic) -is no longer supported. When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) instead. - -If you want to override the job definition (for example, change properties like -`variables`), you need to declare a `container_scanning` job after the -template inclusion and specify any additional keys under it. For example: +If you want to override the job definition (for example, to change properties like `variables`), you +must declare a `container_scanning` job after the template inclusion, and then +specify any additional keys. For example: ```yaml include: @@ -205,15 +190,20 @@ container_scanning: GIT_STRATEGY: fetch ``` +CAUTION: **Deprecated:** +GitLab 13.0 and later doesn't support [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic). +When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) +instead. + ### Vulnerability whitelisting -If you want to whitelist specific vulnerabilities, you'll need to: +To whitelist specific vulnerabilities, follow these steps: -1. Set `GIT_STRATEGY: fetch` in your `.gitlab-ci.yml` file by following the instructions described in the - [overriding the Container Scanning template](#overriding-the-container-scanning-template) section of this document. -1. Define the whitelisted vulnerabilities in a YAML file named `clair-whitelist.yml` which must use the format described - in the [whitelist example file](https://github.com/arminc/clair-scanner/blob/v12/example-whitelist.yaml). -1. Add the `clair-whitelist.yml` file to the Git repository of your project. +1. Set `GIT_STRATEGY: fetch` in your `.gitlab-ci.yml` file by following the instructions in + [overriding the Container Scanning template](#overriding-the-container-scanning-template). +1. Define the whitelisted vulnerabilities in a YAML file named `clair-whitelist.yml`. This must use + the format described in the [whitelist example file](https://github.com/arminc/clair-scanner/blob/v12/example-whitelist.yaml). +1. Add the `clair-whitelist.yml` file to your project's Git repository. ### Running Container Scanning in an offline environment @@ -286,14 +276,13 @@ this with a pipeline means you won't have to do it manually each time. You can u ```yaml image: docker:stable -services: - - docker:19.03.8-dind - stages: - build build_latest_vulnerabilities: stage: build + services: + - docker:19.03.11-dind script: - docker pull arminc/clair-db:latest - docker tag arminc/clair-db:latest $CI_REGISTRY/namespace/clair-vulnerabilities-db @@ -339,11 +328,10 @@ The results are stored in `gl-container-scanning-report.json`. ## Reports JSON format -CAUTION: **Caution:** -The JSON report artifacts are not a public API of Container Scanning and their format may change in the future. +The Container Scanning tool emits a JSON report file. For more information, see the +[schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/container-scanning-report-format.json). -The Container Scanning tool emits a JSON report file. Here is an example of the report structure with all important parts of -it highlighted: +Here's an example Container Scanning report: ```json-doc { @@ -400,49 +388,6 @@ it highlighted: } ``` -CAUTION: **Deprecation:** -Beginning with GitLab 12.9, container scanning no longer reports `undefined` severity and confidence levels. - -Here is the description of the report file structure nodes and their meaning. All fields are mandatory to be present in -the report JSON unless stated otherwise. Presence of optional fields depends on the underlying analyzers being used. - -| Report JSON node | Description | -|------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `version` | Report syntax version used to generate this JSON. | -| `vulnerabilities` | Array of vulnerability objects. | -| `vulnerabilities[].id` | Unique identifier of the vulnerability. | -| `vulnerabilities[].category` | Where this vulnerability belongs (for example, SAST or Container Scanning). For Container Scanning, it will always be `container_scanning`. | -| `vulnerabilities[].message` | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. | -| `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | -| `vulnerabilities[].cve` | (**DEPRECATED - use `vulnerabilities[].id` instead**) A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | -| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) only provides the following levels: `Unknown`, `Low`, `Medium`, `High`, `Critical`. | -| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) does not provide a confidence level, so this value is currently hardcoded to `Unknown`. | -| `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | -| `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | -| `vulnerabilities[].scanner.id` | ID of the scanner as a snake_case string. | -| `vulnerabilities[].scanner.name` | Name of the scanner, for display purposes. | -| `vulnerabilities[].location` | A node that tells where the vulnerability is located. | -| `vulnerabilities[].location.dependency` | A node that describes the dependency of a project where the vulnerability is located. | -| `vulnerabilities[].location.dependency.package` | A node that provides the information on the package where the vulnerability is located. | -| `vulnerabilities[].location.dependency.package.name` | Name of the package where the vulnerability is located. | -| `vulnerabilities[].location.dependency.version` | Version of the vulnerable package. Optional. | -| `vulnerabilities[].location.operating_system` | The operating system that contains the vulnerable package. | -| `vulnerabilities[].location.image` | The Docker image that was analyzed. | -| `vulnerabilities[].identifiers` | An ordered array of references that identify a vulnerability on internal or external DBs. | -| `vulnerabilities[].identifiers[].type` | Type of the identifier. Possible values: common identifier types (among `cve`, `cwe`, `osvdb`, and `usn`). | -| `vulnerabilities[].identifiers[].name` | Name of the identifier for display purpose. | -| `vulnerabilities[].identifiers[].value` | Value of the identifier for matching purpose. | -| `vulnerabilities[].identifiers[].url` | URL to identifier's documentation. Optional. | -| `vulnerabilities[].links` | An array of references to external documentation pieces or articles that describe the vulnerability further. Optional. | -| `vulnerabilities[].links[].name` | Name of the vulnerability details link. Optional. | -| `vulnerabilities[].links[].url` | URL of the vulnerability details document. Optional. | -| `remediations` | An array of objects containing information on cured vulnerabilities along with patch diffs to apply. Empty if no remediations provided by an underlying analyzer. | -| `remediations[].fixes` | An array of strings that represent references to vulnerabilities fixed by this particular remediation. | -| `remediations[].fixes[].id` | The ID of a fixed vulnerability. | -| `remediations[].fixes[].cve` | (**DEPRECATED - use `remediations[].fixes[].id` instead**) A string value that describes a fixed vulnerability in the same format as `vulnerabilities[].cve`. | -| `remediations[].summary` | Overview of how the vulnerabilities have been fixed. | -| `remediations[].diff` | base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). | - ## Security Dashboard The [Security Dashboard](../security_dashboard/index.md) shows you an overview of all @@ -473,7 +418,7 @@ Read more about the [solutions for vulnerabilities](../index.md#solutions-for-vu ## Troubleshooting -### docker: Error response from daemon: failed to copy xattrs +### `docker: Error response from daemon: failed to copy xattrs` When the GitLab Runner uses the Docker executor and NFS is used (for example, `/var/lib/docker` is on an NFS mount), Container Scanning might fail with @@ -486,4 +431,4 @@ docker: Error response from daemon: failed to copy xattrs: failed to set xattr " This is a result of a bug in Docker which is now [fixed](https://github.com/containerd/continuity/pull/138 "fs: add WithAllowXAttrErrors CopyOpt"). To prevent the error, ensure the Docker version that the Runner is using is `18.09.03` or higher. For more information, see -[issue #10241](https://gitlab.com/gitlab-org/gitlab/issues/10241 "Investigate why Container Scanning is not working with NFS mounts"). +[issue #10241](https://gitlab.com/gitlab-org/gitlab/-/issues/10241 "Investigate why Container Scanning is not working with NFS mounts"). diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index cfc679f13a7..256daae46d7 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -1,10 +1,13 @@ --- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- # Dynamic Application Security Testing (DAST) **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4348) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4348) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. NOTE: **4 of the top 6 attacks were application based.** Download our whitepaper, @@ -142,7 +145,14 @@ the site during a scan could lead to inaccurate results. ### Authentication -It's also possible to authenticate the user before performing the DAST checks: +It's also possible to authenticate the user before performing the DAST checks. + +Create masked variables to pass the credentials that DAST will use. +To create masked variables for the username and password, see [Create a custom variable in the UI](../../../ci/variables/README.md#create-a-custom-variable-in-the-ui). +Note that the key of the username variable must be `DAST_USERNAME` +and the key of the password variable must be `DAST_PASSWORD`. + +Other variables that are related to authenticated scans are: ```yaml include: @@ -151,8 +161,6 @@ include: variables: DAST_WEBSITE: https://example.com DAST_AUTH_URL: https://example.com/sign-in - DAST_USERNAME: john.doe@example.com - DAST_PASSWORD: john-doe-password DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional, URLs to skip during the authenticated scan; comma-separated, no spaces in between @@ -439,7 +447,7 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia | Environment variable | Required | Description | |-----------------------------| -----------|--------------------------------------------------------------------------------| -| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address from which to download the analyzer. | +| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address from which to download the analyzer. | | `DAST_WEBSITE` | no| The URL of the website to scan. `DAST_API_SPECIFICATION` must be specified if this is omitted. | | `DAST_API_SPECIFICATION` | no | The API specification to import. `DAST_WEBSITE` must be specified if this is omitted. | | `DAST_AUTH_URL` | no | The authentication URL of the website to scan. Not supported for API scans. | @@ -455,7 +463,15 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia | `DAST_API_HOST_OVERRIDE` | no | Used to override domains defined in API specification files. | | `DAST_EXCLUDE_RULES` | no | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from the scan report. Currently, excluded rules will get executed but the alerts from them will be suppressed. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md). For example, `HTTP Parameter Override` has a rule ID of `10026`. | | `DAST_REQUEST_HEADERS` | no | Set to a comma-separated list of request header names and values. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | -| `DAST_ZAP_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_DEBUG` | no | Enable debug message output. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_SPIDER_MINS` | no | The maximum duration of the spider scan in minutes. Set to zero for unlimited. Defaults to one minute, or unlimited when the scan is a full scan. | +| `DAST_HTML_REPORT` | no | The file name of the HTML report written at the end of a scan. | +| `DAST_MARKDOWN_REPORT` | no | The file name of the Markdown report written at the end of a scan. | +| `DAST_XML_REPORT` | no | The file name of the XML report written at the end of a scan. | +| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | no | Include alpha passive and active scan rules. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_USE_AJAX_SPIDER` | no | Use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | +| `DAST_ZAP_CLI_OPTIONS` | no | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. | +| `DAST_ZAP_LOG_CONFIGURATION` | no | Set to a semicolon-separated list of additional log4j properties for the ZAP Server. For example, `log4j.logger.org.parosproxy.paros.network.HttpSender=DEBUG` | ### DAST command-line options @@ -472,8 +488,9 @@ dast: - /analyze --help ``` -You must then overwrite the `script` command to pass in the appropriate argument. -For example, debug messages can be enabled by using `-d`, as shown in the following configuration: +You must then overwrite the `script` command to pass in the appropriate +argument. For example, passive scanning can be delayed using option `-D`. The following +configuration delays passive scanning by five minutes: ```yaml include: @@ -482,14 +499,14 @@ include: dast: script: - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -d -t $DAST_WEBSITE + - /analyze -D 300 -t $DAST_WEBSITE ``` ### Custom ZAProxy configuration -The ZAProxy server contains many [useful configurable values](https://gitlab.com/gitlab-org/gitlab/issues/36437#note_245801885). +The ZAProxy server contains many [useful configurable values](https://gitlab.com/gitlab-org/gitlab/-/issues/36437#note_245801885). Many key/values for `-config` remain undocumented, but there is an untested list of -[possible keys](https://gitlab.com/gitlab-org/gitlab/issues/36437#note_244981023). +[possible keys](https://gitlab.com/gitlab-org/gitlab/-/issues/36437#note_244981023). Note that these options are not supported by DAST, and may break the DAST scan when used. An example of how to rewrite the Authorization header value with `TOKEN` follows: @@ -497,10 +514,8 @@ when used. An example of how to rewrite the Authorization header value with `TOK include: template: DAST.gitlab-ci.yml -dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -z"-config replacer.full_list\(0\).description=auth -config replacer.full_list\(0\).enabled=true -config replacer.full_list\(0\).matchtype=REQ_HEADER -config replacer.full_list\(0\).matchstr=Authorization -config replacer.full_list\(0\).regex=false -config replacer.full_list\(0\).replacement=TOKEN" -t $DAST_WEBSITE +variables: + DAST_ZAP_CLI_OPTIONS: "-config replacer.full_list(0).description=auth -config replacer.full_list(0).enabled=true -config replacer.full_list(0).matchtype=REQ_HEADER -config replacer.full_list(0).matchstr=Authorization -config replacer.full_list(0).regex=false -config replacer.full_list(0).replacement=TOKEN" ``` ### Cloning the project's repository @@ -508,6 +523,29 @@ dast: The DAST job does not require the project's repository to be present when running, so by default [`GIT_STRATEGY`](../../../ci/yaml/README.md#git-strategy) is set to `none`. +### Debugging DAST jobs + +A DAST job has two executing processes: + +- The ZAP server. +- A series of scripts that start, control and stop the ZAP server. + +Debug mode of the scripts can be enabled by using the `DAST_DEBUG` environment variable. This can help when troubleshooting the job, +and will output statements indicating what percentage of the scan is complete. +For details on using variables, see [Overriding the DAST template](#overriding-the-dast-template). + +Debug mode of the ZAP server can be enabled using the `DAST_ZAP_LOG_CONFIGURATION` environment variable. +The following table outlines examples of values that can be set and the effect that they have on the output that is logged. +Multiple values can be specified, separated by semicolons. + +| Log configuration value | Effect | +|-------------------------------------------------- | ----------------------------------------------------------------- | +| `log4j.rootLogger=DEBUG` | Enable all debug logging statements. | +| `log4j.logger.org.apache.commons.httpclient=DEBUG` | Log every HTTP request and response made by the ZAP server. | +| `log4j.logger.com.crawljax=DEBUG` | Enable Ajax Crawler debug logging statements. | +| `log4j.logger.org.parosproxy.paros=DEBUG` | Enable ZAP server proxy debug logging statements. | +| `log4j.logger.org.zaproxy.zap=DEBUG` | Enable debug logging statements of the general ZAP server code. | + ## Running DAST in an offline environment For self-managed GitLab instances in an environment with limited, restricted, or intermittent access @@ -568,7 +606,8 @@ Alternatively, you can use the variable `SECURE_ANALYZERS_PREFIX` to override th ## Reports -The DAST job can emit various reports. +The DAST tool outputs a report file in JSON format by default. However, this tool can also generate reports in +Markdown, HTML, and XML. For more information, see the [schema for DAST reports](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/dast-report-format.json). ### List of URLs scanned @@ -593,24 +632,22 @@ There are two formats of data in the JSON report that are used side by side: ### Other formats -Reports can also be generated in Markdown, HTML, and XML. - -Reports can be published as artifacts using the following configuration: +Reports can also be generated in Markdown, HTML, and XML. These can be published as artifacts using the following configuration: ```yaml include: template: DAST.gitlab-ci.yml dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -r report.html -w report.md -x report.xml -t $DAST_WEBSITE - - cp /zap/wrk/report.{html,md,xml} "$PWD" + variables: + DAST_HTML_REPORT: report.html + DAST_MARKDOWN_REPORT: report.md + DAST_XML_REPORT: report.xml artifacts: paths: - - report.html - - report.md - - report.xml + - $DAST_HTML_REPORT + - $DAST_MARKDOWN_REPORT + - $DAST_XML_REPORT - gl-dast-report.json ``` @@ -622,18 +659,18 @@ vulnerabilities in your groups, projects and pipelines. Read more about the ## Bleeding-edge vulnerability definitions -ZAProxy first creates rules in the `alpha` class. After a testing period with the -community, they are promoted to `beta`. DAST uses `beta` definitions by default. -To request `alpha` definitions, use `-a` as shown in the following configuration: +ZAP first creates rules in the `alpha` class. After a testing period with +the community, they are promoted to `beta`. DAST uses `beta` definitions by +default. To request `alpha` definitions, use the +`DAST_INCLUDE_ALPHA_VULNERABILITIES` environment variable as shown in the +following configuration: ```yaml include: template: DAST.gitlab-ci.yml -dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -a -t $DAST_WEBSITE +variables: + DAST_INCLUDE_ALPHA_VULNERABILITIES: true ``` ## Interacting with the vulnerabilities @@ -685,16 +722,14 @@ This results in the following error: ``` Fortunately, it's straightforward to increase the amount of memory available -for DAST by overwriting the `script` key in the DAST template: +for DAST by using the `DAST_ZAP_CLI_OPTIONS` environment variable: ```yaml include: - template: DAST.gitlab-ci.yml -dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -t $DAST_WEBSITE -z"-Xmx3072m" +variables: + DAST_ZAP_CLI_OPTIONS: "-Xmx3072m" ``` Here, DAST is being allocated 3072 MB. diff --git a/doc/user/application_security/dependency_list/index.md b/doc/user/application_security/dependency_list/index.md index 73d2cfeaf00..1f730e5f205 100644 --- a/doc/user/application_security/dependency_list/index.md +++ b/doc/user/application_security/dependency_list/index.md @@ -1,6 +1,13 @@ +--- +type: reference, howto +stage: Secure +group: Composition Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Dependency List **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10075) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10075) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0. The Dependency list allows you to see your project's dependencies, and key details about them, including their known vulnerabilities. To see it, @@ -24,8 +31,8 @@ Dependencies are displayed with the following information: | Field | Description | | --------- | ----------- | | Component | The dependency's name and version | -| Packager | The packager used to install the depedency | -| Location | A link to the packager-specific lockfile in your project that declared the dependency | +| Packager | The packager used to install the dependency | +| Location | A link to the packager-specific lock file in your project that declared the dependency | | License | Links to dependency's software licenses | Dependencies shown are initially sorted by the severity of their known vulnerabilities, if any. They @@ -39,7 +46,7 @@ vulnerability, its severity and description then appears below it. ## Licenses -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10536) in GitLab Ultimate 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10536) in GitLab Ultimate 12.3. If the [License Compliance](../../compliance/license_compliance/index.md) CI job is configured, the [discovered licenses](../../compliance/license_compliance/index.md#supported-languages-and-package-managers) will be displayed on this page. diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 53462cf232e..84ec0ec976d 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -1,10 +1,13 @@ --- type: reference, howto +stage: Secure +group: Composition Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Dependency Scanning **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5105) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5105) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. Dependency Scanning helps to automatically find security vulnerabilities in your dependencies while you're developing and testing your applications, such as when your @@ -50,20 +53,27 @@ Beginning with GitLab 13.0, Docker privileged mode is necessary only if you've [ ## Supported languages and package managers -The following languages and dependency managers are supported. - -| Language (package managers) | Supported | Scan tool(s) | -|----------------------------- | --------- | ------------ | -| Java ([Gradle](https://gradle.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Java ([Maven](https://maven.apache.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| JavaScript ([npm](https://www.npmjs.com/), [yarn](https://classic.yarnpkg.com/en/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js/) | -| PHP ([Composer](https://getcomposer.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Python ([pip](https://pip.pypa.io/en/stable/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Python ([Pipfile](https://pipenv.kennethreitz.org/en/latest/basics/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/11756 "Pipfile.lock support for Dependency Scanning"))| not available | -| Python ([poetry](https://python-poetry.org/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/7006 "Support Poetry in Dependency Scanning")) | not available | -| Ruby ([gem](https://rubygems.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) | -| Scala ([sbt](https://www.scala-sbt.org/)) | yes | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | -| Go ([Go Modules](https://github.com/golang/go/wiki/Modules)) | yes ([alpha](https://gitlab.com/gitlab-org/gitlab/issues/7132)) | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +GitLab relies on [`rules`](../../../ci/yaml/README.md#rules) to start relevant analyzers depending on the languages detected in the repository. +The current detection logic limits the maximum search depth to two levels. For example, the `gemnasium-dependency_scanning` job is enabled if a repository contains either a `Gemfile` or `api/Gemfile` file, but not if the only supported dependency file is `api/client/Gemfile`. + +The following languages and dependency managers are supported: + +| Language (package managers) | Supported files | Scan tool(s) | +|----------------------------- | --------------- | ------------ | +| Java ([Gradle](https://gradle.org/), [Maven](https://maven.apache.org/)) | `build.gradle`, `build.gradle.kts`, `pom.xml` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| JavaScript ([npm](https://www.npmjs.com/), [yarn](https://yarnpkg.com/en/)) | `package-lock.json`, `npm-shrinkwrap.json`, `yarn.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [Retire.js](https://retirejs.github.io/retire.js) | +| Go ([Golang](https://golang.org/)) | `go.sum` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| PHP ([Composer](https://getcomposer.org/)) | `composer.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Python ([setuptools](https://setuptools.readthedocs.io/en/latest/), [pip](https://pip.pypa.io/en/stable/), [Pipenv](https://pipenv.pypa.io/en/latest/)) | `setup.py`, `requirements.txt`, `requirements.pip`, `requires.txt`, `Pipfile` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | +| Ruby ([Bundler](https://bundler.io/)) | `Gemfile.lock`, `gems.locked` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) | +| Scala ([sbt](https://www.scala-sbt.org/)) | `build.sbt` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | + +Plans are underway for supporting the following languages, dependency managers, and dependency files. For details, see the issue link for each. + +| Language (package managers) | Supported files | Scan tool(s) | Issue | +|----------------------------- | --------------- | ------------ | ----- | +| Python ([Poetry](https://poetry.eustace.io/)) | `poetry.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#7006](https://gitlab.com/gitlab-org/gitlab/issues/7006) | +| Python ([Pipenv](https://pipenv.pypa.io/en/latest/)) | `Pipfile.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) | [GitLab#11756](https://gitlab.com/gitlab-org/gitlab/-/issues/11756) | ## Contribute your scanner @@ -145,7 +155,7 @@ The following variables allow configuration of global dependency scanning settin | `DS_DEFAULT_ANALYZERS` | Override the names of the official default images. Read more about [customizing analyzers](analyzers.md). | | `DS_DISABLE_DIND` | Disable Docker-in-Docker and run analyzers [individually](#enabling-docker-in-docker). This variable is `true` by default. | | `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs to trust. | -| `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. | +| `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"` | #### Configuring Docker-in-Docker orchestrator @@ -173,9 +183,9 @@ The following variables are used for configuring specific analyzers (used for a | `PIP_INDEX_URL` | `gemnasium-python` | `https://pypi.org/simple` | Base URL of Python Package Index. | | `PIP_EXTRA_INDEX_URL` | `gemnasium-python` | | Array of [extra URLs](https://pip.pypa.io/en/stable/reference/pip_install/#cmdoption-extra-index-url) of package indexes to use in addition to `PIP_INDEX_URL`. Comma-separated. | | `PIP_REQUIREMENTS_FILE` | `gemnasium-python` | | Pip requirements file to be scanned. | -| `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12811) in GitLab 12.7) | -| `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12412) in GitLab 12.2) | -| `DS_PYTHON_VERSION` | `retire.js` | | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12296) in GitLab 12.1)| +| `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | +| `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | +| `DS_PYTHON_VERSION` | `retire.js` | | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12296) in GitLab 12.1)| | `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that will be passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repos). | | `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that will be passed to `gradle` by the analyzer. | | `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer will pass to `sbt`. | @@ -195,7 +205,7 @@ Read more on [how to use private Maven repositories](../index.md#using-private-m ### Enabling Docker-in-Docker -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12487) in GitLab Ultimate 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12487) in GitLab Ultimate 12.5. If needed, you can enable Docker-in-Docker to restore the Dependency Scanning behavior that existed prior to GitLab 13.0. Follow these steps to do so: @@ -244,11 +254,10 @@ the [Dependency List](../dependency_list/index.md). ## Reports JSON format -CAUTION: **Caution:** -The JSON report artifacts are not a public API of Dependency Scanning and their format may change in the future. +The Dependency Scanning tool emits a JSON report file. For more information, see the +[schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/dependency-scanning-report-format.json). -The Dependency Scanning tool emits a JSON report file. Here is an example of the report structure with all important parts of -it highlighted: +Here's an example Dependency Scanning report: ```json-doc { @@ -357,49 +366,6 @@ it highlighted: } ``` -CAUTION: **Deprecation:** -Beginning with GitLab 12.9, dependency scanning no longer reports `undefined` severity and confidence levels. - -This table describes the report file structure nodes and their meaning. All fields are mandatory to be present in -the report JSON, unless stated otherwise. The presence of optional fields depends on the underlying analyzers used. - -| Report JSON node | Description | -|------------------------------------------------------|-------------| -| `version` | Report syntax version used to generate this JSON. | -| `vulnerabilities` | Array of vulnerability objects. | -| `vulnerabilities[].id` | Unique identifier of the vulnerability. | -| `vulnerabilities[].category` | Where this vulnerability belongs, such as SAST or Dependency Scanning. For Dependency Scanning, it will always be `dependency_scanning`. | -| `vulnerabilities[].name` | Name of the vulnerability. Must not include the occurrence's specific information. Optional. | -| `vulnerabilities[].message` | A short text that describes the vulnerability. May include occurrence's specific information. Optional. | -| `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | -| `vulnerabilities[].cve` | (**DEPRECATED - use `vulnerabilities[].id` instead**) A fingerprint string value that represents a concrete occurrence of the vulnerability. Used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | -| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | -| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | -| `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | -| `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | -| `vulnerabilities[].scanner.id` | ID of the scanner as a `snake_case` string. | -| `vulnerabilities[].scanner.name` | Name of the scanner, for display purposes. | -| `vulnerabilities[].location` | A node that tells where the vulnerability is located. | -| `vulnerabilities[].location.file` | Path to the dependencies file (such as `yarn.lock`). Optional. | -| `vulnerabilities[].location.dependency` | A node that describes the dependency of a project where the vulnerability is located. | -| `vulnerabilities[].location.dependency.package` | A node that provides the information on the package where the vulnerability is located. | -| `vulnerabilities[].location.dependency.package.name` | Name of the package where the vulnerability is located. Optional. | -| `vulnerabilities[].location.dependency.version` | Version of the vulnerable package. Optional. | -| `vulnerabilities[].identifiers` | An ordered array of references that identify a vulnerability on internal or external DBs. | -| `vulnerabilities[].identifiers[].type` | Type of the identifier. Possible values: common identifier types (among `cve`, `cwe`, `osvdb`, and `usn`) or analyzer-dependent ones (such as `gemnasium` for [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/)). | -| `vulnerabilities[].identifiers[].name` | Name of the identifier for display purpose. | -| `vulnerabilities[].identifiers[].value` | Value of the identifier for matching purpose. | -| `vulnerabilities[].identifiers[].url` | URL to identifier's documentation. Optional. | -| `vulnerabilities[].links` | An array of references to external documentation pieces or articles that describe the vulnerability further. Optional. | -| `vulnerabilities[].links[].name` | Name of the vulnerability details link. Optional. | -| `vulnerabilities[].links[].url` | URL of the vulnerability details document. Optional. | -| `remediations` | An array of objects containing information on cured vulnerabilities along with patch diffs to apply. Empty if no remediations provided by an underlying analyzer. | -| `remediations[].fixes` | An array of strings that represent references to vulnerabilities fixed by this particular remediation. | -| `remediations[].fixes[].id` | The ID of a fixed vulnerability. | -| `remediations[].fixes[].cve` | (**DEPRECATED - use `remediations[].fixes[].id` instead**) A string value that describes a fixed vulnerability in the same format as `vulnerabilities[].cve`. | -| `remediations[].summary` | Overview of how the vulnerabilities have been fixed. | -| `remediations[].diff` | Base64-encoded remediation code diff, compatible with [`git apply`](https://git-scm.com/docs/git-format-patch#_discussion). | - ## Versioning and release process Please check the [Release Process documentation](https://gitlab.com/gitlab-org/security-products/release/blob/master/docs/release_process.md). @@ -498,42 +464,6 @@ BUNDLER_AUDIT_ADVISORY_DB_REF_NAME: "master" BUNDLER_AUDIT_ADVISORY_DB_URL: "gitlab.example.com/ruby-advisory-db.git" ``` -#### Java (Maven) projects - -When using self-signed certificates, add the following job section to the `.gitlab-ci.yml`: - -```yaml -gemnasium-maven-dependency_scanning: - variables: - MAVEN_CLI_OPTS: "-s settings.xml -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true" -``` - -#### Java (Gradle) projects - -When using self-signed certificates, add the following job section to the `.gitlab-ci.yml`: - -```yaml -gemnasium-maven-dependency_scanning: - before_script: - - echo -n | openssl s_client -connect maven-repo.example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/internal.crt - - keytool -importcert -file /tmp/internal.crt -cacerts -storepass changeit -noprompt -``` - -This adds the self-signed certificates of your Maven repository to the Java KeyStore of the analyzer's Docker image. - -#### Scala (sbt) projects - -When using self-signed certificates, add the following job section to the `.gitlab-ci.yml`: - -```yaml -gemnasium-maven-dependency_scanning: - before_script: - - echo -n | openssl s_client -connect maven-repo.example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/internal.crt - - keytool -importcert -file /tmp/internal.crt -cacerts -storepass changeit -noprompt -``` - -This adds the self-signed certificates of your Maven repository to the Java KeyStore of the analyzer's Docker image. - #### Python (setuptools) When using self-signed certificates for your private PyPi repository, no extra job configuration (aside @@ -543,23 +473,23 @@ ensure that it can reach your private repository. Here is an example configurati 1. Update `setup.py` to create a `dependency_links` attribute pointing at your private repository for each dependency in the `install_requires` list: - ```python - install_requires=['pyparsing>=2.0.3'], - dependency_links=['https://pypi.example.com/simple/pyparsing'], - ``` + ```python + install_requires=['pyparsing>=2.0.3'], + dependency_links=['https://pypi.example.com/simple/pyparsing'], + ``` 1. Fetch the certificate from your repository URL and add it to the project: - ```bash - echo -n | openssl s_client -connect pypi.example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > internal.crt - ``` + ```shell + echo -n | openssl s_client -connect pypi.example.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > internal.crt + ``` 1. Point `setup.py` at the newly downloaded certificate: - ```python - import setuptools.ssl_support - setuptools.ssl_support.cert_paths = ['internal.crt'] - ``` + ```python + import setuptools.ssl_support + setuptools.ssl_support.cert_paths = ['internal.crt'] + ``` ## Limitations @@ -579,9 +509,9 @@ As a workaround, remove the [`retire.js`](analyzers.md#selecting-specific-analyz ## Troubleshooting -### Error response from daemon: error processing tar file: docker-tar: relocation error +### `Error response from daemon: error processing tar file: docker-tar: relocation error` This error occurs when the Docker version that runs the Dependency Scanning job is `19.03.00`. Consider updating to Docker `19.03.1` or greater. Older versions are not affected. Read more in -[this issue](https://gitlab.com/gitlab-org/gitlab/issues/13830#note_211354992 "Current SAST container fails"). +[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/13830#note_211354992 "Current SAST container fails"). diff --git a/doc/user/application_security/img/security_configuration_page_v13_1.png b/doc/user/application_security/img/security_configuration_page_v13_1.png Binary files differindex 0147084f705..176c64a9e87 100644 --- a/doc/user/application_security/img/security_configuration_page_v13_1.png +++ b/doc/user/application_security/img/security_configuration_page_v13_1.png diff --git a/doc/user/application_security/img/vulnerability-check_v13_0.png b/doc/user/application_security/img/vulnerability-check_v13_0.png Binary files differnew file mode 100644 index 00000000000..536fc4f10f7 --- /dev/null +++ b/doc/user/application_security/img/vulnerability-check_v13_0.png diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index eefa52eb5c3..49580f494a2 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -17,14 +17,15 @@ For an overview of application security with GitLab, see ## Quick start -Get started quickly with Dependency Scanning, License Scanning, and Static Application Security -Testing (SAST) by adding the following to your `.gitlab-ci.yml`: +Get started quickly with Dependency Scanning, License Scanning, Static Application Security +Testing (SAST), and Secret Detection by adding the following to your `.gitlab-ci.yml`: ```yaml include: - template: Dependency-Scanning.gitlab-ci.yml - template: License-Scanning.gitlab-ci.yml - template: SAST.gitlab-ci.yml + - template: Secret-Detection.gitlab-ci.yml ``` To add Dynamic Application Security Testing (DAST) scanning, add the following to your @@ -64,14 +65,27 @@ GitLab uses the following tools to scan and report known vulnerabilities found i | [Security Dashboard](security_dashboard/index.md) **(ULTIMATE)** | View vulnerabilities in all your projects and groups. | | [Static Application Security Testing (SAST)](sast/index.md) **(ULTIMATE)** | Analyze source code for known vulnerabilities. | +## Security Scanning with Auto DevOps + +When [Auto DevOps](../../topics/autodevops/) is enabled, all GitLab Security scanning tools will be configured using default settings. + +- [Auto SAST](../../topics/autodevops/stages.md#auto-sast-ultimate) +- [Auto Secret Detection](../../topics/autodevops/stages.md#auto-secret-detection-ultimate) +- [Auto DAST](../../topics/autodevops/stages.md#auto-dast-ultimate) +- [Auto Dependency Scanning](../../topics/autodevops/stages.md#auto-dependency-scanning-ultimate) +- [Auto License Compliance](../../topics/autodevops/stages.md#auto-license-compliance-ultimate) +- [Auto Container Scanning](../../topics/autodevops/stages.md#auto-container-scanning-ultimate) + +While you cannot directly customize Auto DevOps, you can [include the Auto DevOps template in your project's `.gitlab-ci.yml` file](../../topics/autodevops/customize.md#customizing-gitlab-ciyml). + ## Maintenance and update of the vulnerabilities database The scanning tools and vulnerabilities database are updated regularly. | Secure scanning tool | Vulnerabilities database updates | |:-------------------------------------------------------------|-------------------------------------------| -| [Container Scanning](container_scanning/index.md) | Uses `clair`. The latest `clair-db` version is used for each job by running the [`latest` docker image tag](https://gitlab.com/gitlab-org/gitlab/blob/438a0a56dc0882f22bdd82e700554525f552d91b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L37). The `clair-db` database [is updated daily according to the author](https://github.com/arminc/clair-local-scan#clair-server-or-local). | -| [Dependency Scanning](dependency_scanning/index.md) | Relies on `bundler-audit` (for Rubygems), `retire.js` (for NPM packages), and `gemnasium` (GitLab's own tool for all libraries). Both `bundler-audit` and `retire.js` fetch their vulnerabilities data from GitHub repositories, so vulnerabilities added to `ruby-advisory-db` and `retire.js` are immediately available. The tools themselves are updated once per month if there's a new version. The [Gemnasium DB](https://gitlab.com/gitlab-org/security-products/gemnasium-db) is updated at least once a week. See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | +| [Container Scanning](container_scanning/index.md) | Uses `clair`. The latest `clair-db` version is used for each job by running the [`latest` Docker image tag](https://gitlab.com/gitlab-org/gitlab/blob/438a0a56dc0882f22bdd82e700554525f552d91b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml#L37). The `clair-db` database [is updated daily according to the author](https://github.com/arminc/clair-local-scan#clair-server-or-local). | +| [Dependency Scanning](dependency_scanning/index.md) | Relies on `bundler-audit` (for Ruby gems), `retire.js` (for NPM packages), and `gemnasium` (GitLab's own tool for all libraries). Both `bundler-audit` and `retire.js` fetch their vulnerabilities data from GitHub repositories, so vulnerabilities added to `ruby-advisory-db` and `retire.js` are immediately available. The tools themselves are updated once per month if there's a new version. The [Gemnasium DB](https://gitlab.com/gitlab-org/security-products/gemnasium-db) is updated at least once a week. See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | | [Dynamic Application Security Testing (DAST)](dast/index.md) | The scanning engine is updated on a periodic basis. See the [version of the underlying tool `zaproxy`](https://gitlab.com/gitlab-org/security-products/dast/blob/master/Dockerfile#L1). The scanning rules are downloaded at scan runtime. | | [Static Application Security Testing (SAST)](sast/index.md) | Relies exclusively on [the tools GitLab wraps](sast/index.md#supported-languages-and-frameworks). The underlying analyzers are updated at least once per month if a relevant update is available. The vulnerabilities database is updated by the upstream tools. | @@ -82,7 +96,7 @@ tools overrides these tags. The Docker images are updated to match the previous GitLab releases, so users automatically get the latest versions of the scanning tools without having to do anything. There are some known issues with this approach, however, and there is a -[plan to resolve them](https://gitlab.com/gitlab-org/gitlab/issues/9725). +[plan to resolve them](https://gitlab.com/gitlab-org/gitlab/-/issues/9725). ## Interacting with the vulnerabilities @@ -95,7 +109,7 @@ information with several options: - [Dismiss vulnerability](#dismissing-a-vulnerability): Dismissing a vulnerability styles it in strikethrough. - [Create issue](#creating-an-issue-for-a-vulnerability): Create a new issue with the title and - description prepopulated with information from the vulnerability report. By default, such issues + description pre-populated with information from the vulnerability report. By default, such issues are [confidential](../project/issues/confidential_issues.md). - [Solution](#solutions-for-vulnerabilities-auto-remediation): For some vulnerabilities, a solution is provided for how to fix the vulnerability. @@ -142,7 +156,7 @@ button from within the vulnerability modal, or by using the action buttons to th a vulnerability row in the group security dashboard. This creates a [confidential issue](../project/issues/confidential_issues.md) in the project the -vulnerability came from, and prepopulates it with some useful information taken from the vulnerability +vulnerability came from, and pre-populates it with some useful information taken from the vulnerability report. Once the issue is created, you are redirected to it so you can edit, assign, or comment on it. @@ -153,7 +167,7 @@ to the name. ### Solutions for vulnerabilities (auto-remediation) -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5656) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5656) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.7. Some vulnerabilities can be fixed by applying the solution that GitLab automatically generates. The following scanners are supported: @@ -178,7 +192,7 @@ generated by GitLab. To apply the fix: #### Creating a merge request from a vulnerability -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9224) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9224) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.9. In certain cases, GitLab allows you to create a merge request that automatically remediates the vulnerability. Any vulnerability that has a @@ -192,7 +206,7 @@ Click this button to create a merge request to apply the solution onto the sourc ## Security approvals in merge requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9928) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9928) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.2. Merge Request Approvals can be configured to require approval from a member of your security team when a merge request would introduce one of the following security issues: @@ -216,9 +230,15 @@ rating. ### Enabling Security Approvals within a project -To enable Security Approvals, a [project approval rule](../project/merge_requests/merge_request_approvals.md#multiple-approval-rules-premium) +To enable Security Approvals, a [project approval rule](../project/merge_requests/merge_request_approvals.md#adding--editing-a-default-approval-rule) must be created with the case-sensitive name `Vulnerability-Check`. This approval group must be set -with the number of approvals required greater than zero. +with the number of approvals required greater than zero. You must have Maintainer or Owner [permissions](../permissions.md#project-members-permissions) to manage approval rules. + +1. Navigate to your project's **{settings}** **Settings > General** and expand **Merge request approvals**. +1. Click **Add approval rule**, or **Edit**. + - Add or change the **Rule name** to `Vulnerability-Check` (case sensitive). + + Once this group is added to your project, the approval rule is enabled for all merge requests. @@ -236,7 +256,7 @@ An approval is optional when a security report: ## Enabling License Approvals within a project -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13067) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13067) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.3. To enable License Approvals, a [project approval rule](../project/merge_requests/merge_request_approvals.md#multiple-approval-rules-premium) must be created with the case-sensitive name `License-Check`. This approval group must be set @@ -299,7 +319,7 @@ under your project's settings: ## Outdated security reports -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4913) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4913) in GitLab 12.7. When a security report generated for a merge request becomes outdated, the merge request shows a warning message in the security widget and prompts you to take an appropriate action. diff --git a/doc/user/application_security/offline_deployments/index.md b/doc/user/application_security/offline_deployments/index.md index 3deb38902f2..9a2f8768fc0 100644 --- a/doc/user/application_security/offline_deployments/index.md +++ b/doc/user/application_security/offline_deployments/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Secure +group: Static Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Offline environments @@ -49,7 +52,7 @@ you must update each of the scanners to either reference a different, internally-hosted registry or provide access to the individual scanner images. You must also ensure that your app has access to common package repositories -that are not hosted on GitLab.com, such as npm, yarn, or rubygems. Packages +that are not hosted on GitLab.com, such as npm, yarn, or Ruby gems. Packages from these repos can be obtained by temporarily connecting to a network or by mirroring the packages inside your own offline network. diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index 08078a66719..0aa20bf4373 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -1,3 +1,9 @@ +--- +stage: Secure +group: Static Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # SAST Analyzers **(ULTIMATE)** SAST relies on underlying third party tools that are wrapped into what we call @@ -139,7 +145,7 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Analyzers Data -| Property \ Tool | Apex | Bandit | Brakeman | ESLint security | Find Sec Bugs | Flawfinder | Gosec | Kubesec Scanner | NodeJsScan | Php CS Security Audit | Security code Scan (.NET) | Sobelow | TSLint Security | +| Property \ Tool | Apex | Bandit | Brakeman | ESLint security | Find Sec Bugs | Flawfinder | Gosec | Kubesec Scanner | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Sobelow | TSLint Security | | --------------------------------------- | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :---------------------: | :-------------------------: | :----------------: | :-------------: | | Severity | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | ✓ | | Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 370c6d0e8e7..a5497e3d38c 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -1,10 +1,13 @@ --- +stage: Secure +group: Static Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: reference, howto --- # Static Application Security Testing (SAST) **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3775) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3775) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. NOTE: **4 of the top 6 attacks were application based.** Download our whitepaper, @@ -71,10 +74,11 @@ The following table shows which languages, package managers and frameworks are s | .NET Framework | [Security Code Scan](https://security-code-scan.github.io) | 13.0 | | Any | [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) | 11.9 | | Apex (Salesforce) | [PMD](https://pmd.github.io/pmd/index.html) | 12.1 | -| C/C++ | [Flawfinder](https://dwheeler.com/flawfinder/) | 10.7 | +| C/C++ | [Flawfinder](https://github.com/david-a-wheeler/flawfinder) | 10.7 | | Elixir (Phoenix) | [Sobelow](https://github.com/nccgroup/sobelow) | 11.10 | | Go | [Gosec](https://github.com/securego/gosec) | 10.7 | | Groovy ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 11.3 (Gradle) & 11.9 (Ant, Maven, SBT) | +| Helm Charts | [Kubesec](https://github.com/controlplaneio/kubesec) | 13.1 | | Java ([Ant](https://ant.apache.org/), [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) and [SBT](https://www.scala-sbt.org/)) | [SpotBugs](https://spotbugs.github.io/) with the [find-sec-bugs](https://find-sec-bugs.github.io/) plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (Ant, SBT) | | JavaScript | [ESLint security plugin](https://github.com/nodesecurity/eslint-plugin-security) | 11.8 | | Kubernetes manifests | [Kubesec](https://github.com/controlplaneio/kubesec) | 12.6 | @@ -196,7 +200,7 @@ jobs. #### Enabling Kubesec analyzer -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12752) in GitLab Ultimate 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12752) in GitLab Ultimate 12.6. You need to set `SCAN_KUBERNETES_MANIFESTS` to `"true"` to enable the Kubesec analyzer. In `.gitlab-ci.yml`, define: @@ -285,8 +289,8 @@ Some analyzers make it possible to filter out vulnerabilities under a given thre | Environment variable | Default value | Description | |-------------------------|---------------|-------------| -| `SAST_EXCLUDED_PATHS` | - | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories will also match patterns. | -| `SAST_BANDIT_EXCLUDED_PATHS` | - | comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html) | +| `SAST_EXCLUDED_PATHS` | `spec, test, tests, tmp` | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories will also match patterns. | +| `SAST_BANDIT_EXCLUDED_PATHS` | - | comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html); For example: `'*/tests/*'` | | `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | | `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | | `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | @@ -313,19 +317,22 @@ Some analyzers can be customized with environment variables. | Environment variable | Analyzer | Description | |-----------------------------|----------|-------------| -| `SCAN_KUBERNETES_MANIFESTS` | Kubesec | Set to `"true"` to scan Kubernetes manifests. | -| `ANT_HOME` | SpotBugs | The `ANT_HOME` environment variable. | -| `ANT_PATH` | SpotBugs | Path to the `ant` executable. | -| `GRADLE_PATH` | SpotBugs | Path to the `gradle` executable. | -| `JAVA_OPTS` | SpotBugs | Additional arguments for the `java` executable. | -| `JAVA_PATH` | SpotBugs | Path to the `java` executable. | -| `SAST_JAVA_VERSION` | SpotBugs | Which Java version to use. Supported versions are `8` and `11`. Defaults to `8`. | -| `MAVEN_CLI_OPTS` | SpotBugs | Additional arguments for the `mvn` or `mvnw` executable. | -| `MAVEN_PATH` | SpotBugs | Path to the `mvn` executable. | -| `MAVEN_REPO_PATH` | SpotBugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | -| `SBT_PATH` | SpotBugs | Path to the `sbt` executable. | -| `FAIL_NEVER` | SpotBugs | Set to `1` to ignore compilation failure. | -| `SAST_GOSEC_CONFIG` | Gosec | Path to configuration for Gosec (optional). | +| `SCAN_KUBERNETES_MANIFESTS` | Kubesec | Set to `"true"` to scan Kubernetes manifests. | +| `KUBESEC_HELM_CHARTS_PATH` | Kubesec | Optional path to Helm charts that `helm` will use to generate a Kubernetes manifest that `kubesec` will scan. If dependencies are defined, `helm dependency build` should be ran in a `before_script` to fetch the necessary dependencies. | +| `KUBESEC_HELM_OPTIONS` | Kubesec | Additional arguments for the `helm` executable. | +| `ANT_HOME` | SpotBugs | The `ANT_HOME` environment variable. | +| `ANT_PATH` | SpotBugs | Path to the `ant` executable. | +| `GRADLE_PATH` | SpotBugs | Path to the `gradle` executable. | +| `JAVA_OPTS` | SpotBugs | Additional arguments for the `java` executable. | +| `JAVA_PATH` | SpotBugs | Path to the `java` executable. | +| `SAST_JAVA_VERSION` | SpotBugs | Which Java version to use. Supported versions are `8` and `11`. Defaults to `8`. | +| `MAVEN_CLI_OPTS` | SpotBugs | Additional arguments for the `mvn` or `mvnw` executable. | +| `MAVEN_PATH` | SpotBugs | Path to the `mvn` executable. | +| `MAVEN_REPO_PATH` | SpotBugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | +| `SBT_PATH` | SpotBugs | Path to the `sbt` executable. | +| `FAIL_NEVER` | SpotBugs | Set to `1` to ignore compilation failure. | +| `SAST_GOSEC_CONFIG` | Gosec | Path to configuration for Gosec (optional). | +| `PHPCS_SECURITY_AUDIT_PHP_EXTENSIONS` | phpcs-security-audit | Comma separated list of additional PHP Extensions. | #### Custom environment variables @@ -342,11 +349,10 @@ analyzer containers: `DOCKER_`, `CI`, `GITLAB_`, `FF_`, `HOME`, `PWD`, `OLDPWD`, ## Reports JSON format -CAUTION: **Caution:** -The JSON report artifacts are not a public API of SAST and their format may change in the future. +The SAST tool emits a JSON report file. For more information, see the +[schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/sast-report-format.json). -The SAST tool emits a JSON report file. Here is an example of the report structure with all important parts of -it highlighted: +Here's an example SAST report: ```json-doc { @@ -421,40 +427,6 @@ it highlighted: } ``` -CAUTION: **Deprecation:** -Beginning with GitLab 12.9, SAST no longer reports `undefined` severity and confidence levels. - -Here is the description of the report file structure nodes and their meaning. All fields are mandatory in -the report JSON unless stated otherwise. Presence of optional fields depends on the underlying analyzers being used. - -| Report JSON node | Function | -|-----------------------------------------|----------| -| `version` | Report syntax version used to generate this JSON. | -| `vulnerabilities` | Array of vulnerability objects. | -| `vulnerabilities[].id` | Unique identifier of the vulnerability. | -| `vulnerabilities[].category` | Where this vulnerability belongs (such as SAST, Dependency Scanning). For SAST, it will always be `sast`. | -| `vulnerabilities[].name` | Name of the vulnerability. Must not include the occurrence's specific information. Optional. | -| `vulnerabilities[].message` | A short text that describes the vulnerability, it may include the occurrence's specific information. Optional. | -| `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | -| `vulnerabilities[].cve` | (**DEPRECATED - use `vulnerabilities[].id` instead**) A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | -| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | -| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | -| `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | -| `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | -| `vulnerabilities[].scanner.id` | ID of the scanner as a snake_case string. | -| `vulnerabilities[].scanner.name` | Name of the scanner, for display purposes. | -| `vulnerabilities[].location` | A node that tells where the vulnerability is located. | -| `vulnerabilities[].location.file` | Path to the file where the vulnerability is located. Optional. | -| `vulnerabilities[].location.start_line` | The first line of the code affected by the vulnerability. Optional. | -| `vulnerabilities[].location.end_line` | The last line of the code affected by the vulnerability. Optional. | -| `vulnerabilities[].location.class` | If specified, provides the name of the class where the vulnerability is located. Optional. | -| `vulnerabilities[].location.method` | If specified, provides the name of the method where the vulnerability is located. Optional. | -| `vulnerabilities[].identifiers` | An ordered array of references that identify a vulnerability on internal or external databases. | -| `vulnerabilities[].identifiers[].type` | Type of the identifier. Possible values: common identifier types (among `cve`, `cwe`, `osvdb`, and `usn`) or analyzer-dependent ones (like `bandit_test_id` for [Bandit analyzer](https://wiki.openstack.org/wiki/Security/Projects/Bandit)). | -| `vulnerabilities[].identifiers[].name` | Name of the identifier for display purposes. | -| `vulnerabilities[].identifiers[].value` | Value of the identifier for matching purposes. | -| `vulnerabilities[].identifiers[].url` | URL to identifier's documentation. Optional. | - ## Secret detection Learn more about [Secret Detection](../secret_detection). @@ -513,7 +485,6 @@ registry.gitlab.com/gitlab-org/security-products/analyzers/bandit:2 registry.gitlab.com/gitlab-org/security-products/analyzers/brakeman:2 registry.gitlab.com/gitlab-org/security-products/analyzers/eslint:2 registry.gitlab.com/gitlab-org/security-products/analyzers/flawfinder:2 -registry.gitlab.com/gitlab-org/security-products/analyzers/go-ast-scanner:2 registry.gitlab.com/gitlab-org/security-products/analyzers/gosec:2 registry.gitlab.com/gitlab-org/security-products/analyzers/kubesec:2 registry.gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan:2 @@ -553,9 +524,9 @@ security reports without requiring internet access. ## Troubleshooting -### Error response from daemon: error processing tar file: docker-tar: relocation error +### `Error response from daemon: error processing tar file: docker-tar: relocation error` This error occurs when the Docker version that runs the SAST job is `19.03.0`. Consider updating to Docker `19.03.1` or greater. Older versions are not affected. Read more in -[this issue](https://gitlab.com/gitlab-org/gitlab/issues/13830#note_211354992 "Current SAST container fails"). +[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/13830#note_211354992 "Current SAST container fails"). diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 74dd3c89984..85933c31a34 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Secure +group: Static Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Secret Detection **(ULTIMATE)** @@ -26,18 +29,30 @@ GitLab displays identified secrets as part of the SAST reports visibly in a few ## Use cases -- Detecting accidental commit of secrets like keys, passwords, and API tokens. +- Detecting unintentional commit of secrets like keys, passwords, and API tokens. - Performing a single or recurring scan of the full history of your repository for secrets. +## Requirements + +To run Secret Detection jobs, by default, you need GitLab Runner with the +[`docker`](https://docs.gitlab.com/runner/executors/docker.html) or +[`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. +If you're using the shared Runners on GitLab.com, this is enabled by default. + +CAUTION: **Caution:** Our Secret Detection jobs currently expect a Linux container type. Windows containers are not yet supported. + +CAUTION: **Caution:** +If you use your own Runners, make sure the Docker version installed +is **not** `19.03.0`. See [troubleshooting information](../sast#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details. + ## Configuration -If you already have SAST enabled for your app, you don’t need to take any action to benefit from this -new feature. It is also included in the Auto DevOps default configuration. +NOTE: **Note:** +With GitLab 13.1 Secret Detection was split into its own CI/CD template. -Secret Detection is performed by a [specific analyzer](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml#L180) -during the `sast` job. It runs regardless of the programming -language of your app, and you don't need to change your -CI/CD configuration file to enable it. Results are available in the SAST report. +Secret Detection is performed by a [specific analyzer](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) +during the `secret-detection` job. It runs regardless of the programming +language of your app. The Secret Detection analyzer includes [Gitleaks](https://github.com/zricethezav/gitleaks) and [TruffleHog](https://github.com/dxa4481/truffleHog) checks. @@ -47,15 +62,99 @@ with a dollar sign (`$`) as this likely indicates the password being used is an variable. For example, `https://username:$password@example.com/path/to/repo` won't be detected, whereas `https://username:password@example.com/path/to/repo` would be detected. +NOTE: **Note:** +You don't have to configure Secret Detection manually as shown in this section if you're using [Auto Secret Detection](../../../topics/autodevops/stages.md#auto-secret-detection-ultimate) +provided by [Auto DevOps](../../../topics/autodevops/index.md). + +To enable Secret Detection for GitLab 13.1 and later, you must include the `Secret-Detection.gitlab-ci.yml` template that’s provided as a part of your GitLab installation. For GitLab versions earlier than 11.9, you can copy and use the job as defined in that template. + +Add the following to your `.gitlab-ci.yml` file: + +```yaml +include: + - template: Secret-Detection.gitlab-ci.yml +``` + +The included template creates Secret Detection jobs in your CI/CD pipeline and scans +your project's source code for secrets. + +The results are saved as a +[Secret Detection report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportssecret_detection-ultimate) +that you can later download and analyze. Due to implementation limitations, we +always take the latest Secret Detection artifact available. + +### Using the SAST Template + +Prior to GitLab 13.1, Secret Detection was part of [SAST configuration](../sast#configuration). +If you already have SAST enabled for your app configured before GitLab 13.1, +you don't need to manually configure it. + +CAUTION: **Planned Deprecation:** +In a future GitLab release, configuring Secret Detection with the SAST template will be deprecated. Please begin using `Secret-Detection.gitlab-ci.yml` +to prevent future issues. We have made a +[video to guide you through the process of transitioning](https://www.youtube.com/watch?v=W2tjcQreDwQ) +to this new template. + +<div class="video-fallback"> + See the video: <a href="https://www.youtube.com/watch?v=W2tjcQreDwQ">Walkthrough of historical secret scan</a>. +</div> +<figure class="video-container"> + <iframe src="https://www.youtube.com/embed/W2tjcQreDwQ" frameborder="0" allowfullscreen="true"> </iframe> +</figure> + +When using the SAST template, Secret Detection is performed by a [specific analyzer](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml#L180) +during the `sast` job. It runs regardless of the programming +language of your app, and you don't need to change your +CI/CD configuration file to enable it. Results are available in the SAST report. + +### Customizing settings + +The Secret Detection scan settings can be changed through [environment variables](#available-variables) +by using the +[`variables`](../../../ci/yaml/README.md#variables) parameter in `.gitlab-ci.yml`. + +To override a job definition, (for example, change properties like `variables` or `dependencies`), +declare a job with the same name as the SAST job to override. Place this new job after the template +inclusion and specify any additional keys under it. + +In the following example, we include the Secret Detection template and at the same time we +override the `secret-scan` job with the `SECRET_DETECTION_HISTORIC_SCAN` variable to `true`: + +```yaml +include: + - template: Secret-Detection.gitlab-ci.yml + +secrets-scan: + variables: + SECRET_DETECTION_HISTORIC_SCAN: true +``` + +Because the template is [evaluated before](../../../ci/yaml/README.md#include) +the pipeline configuration, the last mention of the variable takes precedence. + +CAUTION: **Deprecation:** +Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/README.md#onlyexcept-basic) +is no longer supported. When overriding the template, you must use [`rules`](../../../ci/yaml/README.md#rules) instead. + +#### Available variables + +Secret Detection can be customized by defining available variables: + +| Environment variable | Default value | Description | +|-------------------------|---------------|-------------| +| `SECRET_DETECTION_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. | +| `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. | +| `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | + ## Full History Secret Scan -GitLab 12.11 introduced support for scanning the full history of a reposity. This new functionality +GitLab 12.11 introduced support for scanning the full history of a repository. This new functionality is particularly useful when you are enabling Secret Detection in a repository for the first time and you want to perform a full secret scan. Running a secret scan on the full history can take a long time, especially for larger repositories with lengthy Git histories. We recommend not setting this variable -as part of your normal job defintion. +as part of your normal job definition. -A new configuration variable ([`SAST_GITLEAKS_HISTORIC_SCAN`](../sast/#vulnerability-filters)) +A new configuration variable ([`SECRET_DETECTION_HISTORIC_SCAN`](../sast/#vulnerability-filters)) can be set to change the behavior of the GitLab Secret Detection scan to run on the entire Git history of a repository. We have created a [short video walkthrough](https://youtu.be/wDtc_K00Y0A) showcasing how you can perform a full history secret scan. diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png Binary files differnew file mode 100644 index 00000000000..0dfe7b637cd --- /dev/null +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_export_csv_v13_1.png diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png Binary files differindex c788e2165ad..4c7b5cc724f 100644 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v13_0.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png Binary files differindex bb88b7f371c..878bb83c2a2 100644 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v13_0.png diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 2988b3642ef..60798b9c921 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -36,7 +36,7 @@ To use the instance, group, project, or pipeline security dashboard: ## Pipeline Security -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13496) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13496) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.3. At the pipeline level, the Security section displays the vulnerabilities present in the branch of the project the pipeline was run against. @@ -49,7 +49,7 @@ A pipeline consists of multiple jobs, including SAST and DAST scanning. If any j ## Project Security Dashboard -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6165) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6165) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.1. At the project level, the Security Dashboard displays the latest security reports for your project. Use it to find and fix vulnerabilities. @@ -70,7 +70,7 @@ of thousands of vulnerabilities. Do not close the page until the download finish ## Group Security Dashboard -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6709) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6709) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.5. The group Security Dashboard gives an overview of the vulnerabilities of all the projects in a group and its subgroups. @@ -120,9 +120,24 @@ vulnerabilities are not included either. Read more on how to [interact with the vulnerabilities](../index.md#interacting-with-the-vulnerabilities). +### Export vulnerabilities + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213013) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. + +You can export all your vulnerabilities as CSV by clicking the **{upload}** **Export** button +located at the top right of the **Group Security Dashboard**. After the report builds, the CSV +report downloads to your local machine. The report contains all vulnerabilities for the projects +defined in the **Group Security Dashboard**, as filters don't apply to the export function. + +NOTE: **Note:** +It may take several minutes for the download to start if your project contains thousands of +vulnerabilities. Don't close the page until the download finishes. + + + ## Instance Security Dashboard -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6953) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6953) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.8. At the instance level, the Security Dashboard displays the vulnerabilities present in all of the projects that you have added to it. It includes all diff --git a/doc/user/application_security/threat_monitoring/index.md b/doc/user/application_security/threat_monitoring/index.md index 7bd148edd15..434048896fe 100644 --- a/doc/user/application_security/threat_monitoring/index.md +++ b/doc/user/application_security/threat_monitoring/index.md @@ -1,14 +1,18 @@ --- type: reference, howto +stage: Defend +group: Container Security +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Threat Monitoring **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14707) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14707) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. -The **Threat Monitoring** page provides metrics for the GitLab -application runtime security features. You can access these metrics by -navigating to your project's **Security & Compliance > Threat Monitoring** page. +The **Threat Monitoring** page provides metrics and policy management +for the GitLab application runtime security features. You can access +these by navigating to your project's **Security & Compliance > Threat +Monitoring** page. GitLab supports statistics for the following security features: @@ -42,7 +46,7 @@ investigate it for potential threats by ## Container Network Policy -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32365) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32365) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. The **Container Network Policy** section provides packet flow metrics for your application's Kubernetes namespace. This section has the following @@ -74,3 +78,41 @@ about your packet flow: If a significant percentage of packets is dropped, you should investigate it for potential threats by [examining the Cilium logs](../../clusters/applications.md#install-cilium-using-gitlab-cicd). + +## Container Network Policy management + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3328) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. + +The **Threat Monitoring** page's **Policy** tab displays deployed +network policies for all available environments. You can check a +network policy's `yaml` manifest and toggle the policy's enforcement +status. This section has the following prerequisites: + +- Your project contains at least one [environment](../../../ci/environments/index.md) +- You've [installed Cilium](../../clusters/applications.md#install-cilium-using-gitlab-cicd) + +Network policies are fetched directly from the selected environment's +deployment platform. Changes performed outside of this tab are +reflected upon refresh. Enforcement status changes are deployed +directly to a deployment namespace of the selected environment. + +NOTE: **Note:** +If you're using [Auto DevOps](../../../topics/autodevops/index.md) and +change a policy in this section, your `auto-deploy-values.yaml` file +doesn't update. Auto DevOps users must make changes by following +the [Container Network Policy documentation](../../../topics/autodevops/stages.md#network-policy). + +### Changing enforcement status + +To change a network policy's enforcement status: + +- Click the network policy you want to update. +- Click the **Enforcement status** toggle to update the selected policy. +- Click the **Apply changes** button to deploy network policy changes. + +NOTE: **Note:** +Disabled network policies have the +`network-policy.gitlab.com/disabled_by: gitlab` selector inside the +`podSelector` block. This narrows the scope of such a policy and as a +result it doesn't affect any pods. The policy itself is still deployed +to the corresponding deployment namespace. diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_download_patch_button_v13_1.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_download_patch_button_v13_1.png Binary files differnew file mode 100644 index 00000000000..b925c342a11 --- /dev/null +++ b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_download_patch_button_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_dropdown_v13_1.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_dropdown_v13_1.png Binary files differnew file mode 100644 index 00000000000..2063762d3eb --- /dev/null +++ b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_dropdown_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_v13_1.png b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_v13_1.png Binary files differnew file mode 100644 index 00000000000..ee4e97bcafe --- /dev/null +++ b/doc/user/application_security/vulnerabilities/img/standalone_vulnerability_page_merge_request_button_v13_1.png diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index b691a97fc32..b3128e49980 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -1,10 +1,13 @@ --- type: reference, howto +stage: Secure +group: Vulnerability Research +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Standalone Vulnerability pages -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13561) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13561) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. Each security vulnerability in the [Vulnerability List](../dependency_list/index.md) has its own standalone page. @@ -17,7 +20,7 @@ several different ways: - [Change the Vulnerability Status](#changing-vulnerability-status) - You can change the status of a vulnerability to **Detected**, **Confirmed**, **Dismissed**, or **Resolved**. - [Create issue](#creating-an-issue-for-a-vulnerability) - Create a new issue with the - title and description prepopulated with information from the vulnerability report. + title and description pre-populated with information from the vulnerability report. By default, such issues are [confidential](../../project/issues/confidential_issues.md). - [Solution](#automatic-remediation-solutions-for-vulnerabilities) - For some vulnerabilities, a solution is provided for how to fix the vulnerability. @@ -39,7 +42,7 @@ the following values: You can create an issue for a vulnerability by selecting the **Create issue** button. This creates a [confidential issue](../../project/issues/confidential_issues.md) in the -project the vulnerability came from, and prepopulates it with useful information from +project the vulnerability came from, and pre-populates it with useful information from the vulnerability report. After the issue is created, GitLab redirects you to the issue page so you can edit, assign, or comment on the issue. @@ -52,14 +55,19 @@ generates for you. GitLab supports the following scanners: is only available for Node.js projects managed with `yarn`. - [Container Scanning](../container_scanning/index.md). +When an automatic solution is available, the button in the header will show "Resolve with merge request": + + + +Selecting the button will create a merge request with the automatic solution. + ### Manually applying a suggested patch -To apply a patch automatically generated by GitLab to fix a vulnerability: +To manually apply the patch that was generated by GitLab for a vulnerability, select the dropdown arrow on the "Resolve +with merge request" button, then select the "Download patch to resolve" option: + + -1. Open the issue created in [Create issue](#creating-an-issue-for-a-vulnerability). -1. In the **Issue description**, scroll to **Solution** and download the linked patch file. -1. Ensure your local project has the same commit checked out that was used to generate the patch. -1. Run `git apply remediation.patch` to apply the patch. -1. Verify and commit the changes to your branch. +This will change the button text to "Download patch to resolve". Click on it to download the patch: - + diff --git a/doc/user/award_emojis.md b/doc/user/award_emojis.md index 5a82712c055..47249a44bc1 100644 --- a/doc/user/award_emojis.md +++ b/doc/user/award_emojis.md @@ -1,9 +1,15 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Award emoji -> - First [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1825) in GitLab 8.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/1825) in GitLab 8.2. > - GitLab 9.0 [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9570) the usage of native emoji if the platform -> supports them and falls back to images or CSS sprites. This change greatly -> improved award emoji performance overall. +> supports them and falls back to images or CSS sprites. This change greatly +> improved award emoji performance overall. When you're collaborating online, you get fewer opportunities for high-fives and thumbs-ups. Emoji can be awarded to [issues](project/issues/index.md), [merge requests](project/merge_requests/index.md), diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 9ede9d9fdef..86624d12bcf 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -332,7 +332,7 @@ Updating [Ingress](#ingress) to the most recent version enables you to take adva ##### Viewing Web Application Firewall traffic -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14707) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14707) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. You can view Web Application Firewall traffic by navigating to your project's **Security & Compliance > Threat Monitoring** page. @@ -458,7 +458,7 @@ file. ### Crossplane -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34702) in GitLab 12.5 for project-level clusters. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34702) in GitLab 12.5 for project-level clusters. [Crossplane](https://crossplane.github.io/docs/v0.9/) is a multi-cloud control plane useful for managing applications and infrastructure across multiple clouds. It extends the @@ -483,7 +483,7 @@ For information on configuring Crossplane installed on the cluster, see [Crossplane configuration](crossplane.md). NOTE: **Note:** -[`alpha/crossplane`](https://charts.crossplane.io/alpha/) chart v0.4.1 is used to +[`alpha/crossplane`](https://github.com/crossplane/crossplane/tree/v0.4.1/cluster/charts/crossplane) chart v0.4.1 is used to install Crossplane using the [`values.yaml`](https://github.com/crossplane/crossplane/blob/master/cluster/charts/crossplane/values.yaml.tmpl) file. @@ -615,6 +615,8 @@ Supported applications: - [Crossplane](#install-crossplane-using-gitlab-cicd) - [Fluentd](#install-fluentd-using-gitlab-cicd) - [Knative](#install-knative-using-gitlab-cicd) +- [PostHog](#install-posthog-using-gitlab-cicd) +- [Prometheus](#install-prometheus-using-gitlab-cicd) ### Usage @@ -779,6 +781,77 @@ postgresql: postgresqlPassword: example-postgresql-password ``` +### Install PostHog using GitLab CI/CD + +[PostHog](https://www.posthog.com) 🦔 is a developer-friendly, open-source product analytics platform. + +To install PostHog into the `gitlab-managed-apps` namespace of your cluster, +define the `.gitlab/managed-apps/config.yaml` file with: + +```yaml +posthog: + installed: true +``` + +You can customize the installation of PostHog by defining `.gitlab/managed-apps/posthog/values.yaml` +in your cluster management project. Refer to the [Configuration section of the PostHog chart's README](https://github.com/PostHog/charts/tree/master/charts/posthog) +for the available configuration options. + +NOTE: **Note:** +You must provide a PostgreSQL password in `postgresql.postgresqlPassword` +or you will receive authentication errors. +See the [PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) for more information. + +Redis pods are restarted between upgrades. To prevent downtime, provide a Redis +password using the `redis.password` key. This prevents a new password from +being generated on each restart. + +Here is an example configuration for PostHog: + +```yaml +ingress: + enabled: true + hostname: "<posthog.example.com>" + +# This will be autogenerated if you skip it. Include if you have 2 or more web replicas +posthogSecret: 'long-secret-key-used-to-sign-cookies' + +# Needs to be here between runs. +# See https://github.com/helm/charts/tree/master/stable/postgresql#upgrade for more info +postgresql: + postgresqlPassword: example-postgresql-password + +# Recommended to set this to a value to redis prevent downtime between upgrades +redis: + password: example-redis-password +``` + +NOTE: **Note:** +Support for the PostHog managed application is provided by the PostHog team. +If you run into issues, please [open a support ticket](https://github.com/PostHog/posthog/issues/new/choose) directly. + +### Install Prometheus using GitLab CI/CD + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25138) in GitLab 12.8. + +[Prometheus](https://prometheus.io/docs/introduction/overview/) is an +open-source monitoring and alerting system for supervising your +deployed applications. + +To install Prometheus into the `gitlab-managed-apps` namespace of your cluster, +define the `.gitlab/managed-apps/config.yaml` file with: + +```yaml +prometheus: + installed: true +``` + +You can customize the installation of Prometheus by defining +`.gitlab/managed-apps/prometheus/values.yaml` in your cluster management +project. Refer to the +[Configuration section of the Prometheus chart's README](https://github.com/helm/charts/tree/master/stable/prometheus#configuration) +for the available configuration options. + ### Install GitLab Runner using GitLab CI/CD GitLab Runner is installed using GitLab CI/CD by defining configuration in @@ -857,10 +930,10 @@ Major upgrades might require additional setup steps, please consult the official [upgrade guide](https://docs.cilium.io/en/stable/install/upgrade/) for more information. -By default, Cilium will drop all non-whitelisted packets upon policy +By default, Cilium will drop all disallowed packets upon policy deployment. The audit mode is scheduled for release in [Cilium 1.8](https://github.com/cilium/cilium/pull/9970). In the audit -mode, non-whitelisted packets will not be dropped, and audit +mode, disallowed packets will not be dropped, and audit notifications will be generated instead. GitLab provides alternative Docker images for Cilium with the audit patch included. You can switch to the custom build and enable the audit mode by adding the following to @@ -915,7 +988,7 @@ metrics: ### Install Vault using GitLab CI/CD -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9982) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9982) in GitLab 12.9. [Hashicorp Vault](https://www.vaultproject.io/) is a secrets management solution which can be used to safely manage and store passwords, credentials, certificates and more. A Vault @@ -940,17 +1013,17 @@ when upgrading the Vault application. To optimally use Vault in a production environment, it's ideal to have a good understanding of the internals of Vault and how to configure it. This can be done by reading the -[the Vault documentation](https://www.vaultproject.io/docs/internals/) as well as +[the Vault documentation](https://www.vaultproject.io/docs/internals) as well as the Vault Helm chart [`values.yaml` file](https://github.com/hashicorp/vault-helm/blob/v0.3.3/values.yaml). At a minimum you will likely set up: -- A [seal](https://www.vaultproject.io/docs/configuration/seal/) for extra encryption +- A [seal](https://www.vaultproject.io/docs/configuration/seal) for extra encryption of the master key. -- A [storage backend](https://www.vaultproject.io/docs/configuration/storage/) that is +- A [storage backend](https://www.vaultproject.io/docs/configuration/storage) that is suitable for environment and storage security requirements. -- [HA Mode](https://www.vaultproject.io/docs/concepts/ha/). -- [The Vault UI](https://www.vaultproject.io/docs/configuration/ui/). +- [HA Mode](https://www.vaultproject.io/docs/concepts/ha). +- [The Vault UI](https://www.vaultproject.io/docs/configuration/ui). The following is an example values file (`.gitlab/managed-apps/vault/values.yaml`) that configures Google Key Management Service for auto-unseal, using a Google Cloud Storage backend, enabling @@ -1048,12 +1121,12 @@ You can customize the installation of JupyterHub by defining a `.gitlab/managed-apps/jupyterhub/values.yaml` file in your cluster management project. Refer to the -[chart reference](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference.html) for the +[chart reference](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html) for the available configuration options. ### Install Elastic Stack using GitLab CI/CD -> [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/45) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25138) in GitLab 12.8. Elastic Stack is installed using GitLab CI/CD by defining configuration in `.gitlab/managed-apps/config.yaml`. @@ -1080,7 +1153,7 @@ In this alpha implementation of installing Elastic Stack through CI, reading the ### Install Crossplane using GitLab CI/CD -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/35675) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35675) in GitLab 12.9. Crossplane is installed using GitLab CI/CD by defining configuration in `.gitlab/managed-apps/config.yaml`. @@ -1157,7 +1230,7 @@ GitLab provides [Invocation Metrics](../project/clusters/serverless/index.md#inv 1. Knative and Prometheus managed applications installed on your cluster. 1. Manually applied the custom metrics on your cluster by running the following command: - ```bash + ```shell kubectl apply -f https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/raw/02c8231e30ef5b6725e6ba368bc63863ceb3c07d/src/default-data/knative/istio-metrics.yaml ``` @@ -1166,7 +1239,7 @@ GitLab provides [Invocation Metrics](../project/clusters/serverless/index.md#inv To uninstall Knative, you must first manually remove any custom metrics you have added by running the following command: -```bash +```shell kubectl delete -f https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/raw/02c8231e30ef5b6725e6ba368bc63863ceb3c07d/src/default-data/knative/istio-metrics.yaml ``` @@ -1197,7 +1270,7 @@ chart plus the values set by ## Uninstalling applications -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/60665) in GitLab 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/60665) in GitLab 11.11. The applications below can be uninstalled. diff --git a/doc/user/clusters/crossplane.md b/doc/user/clusters/crossplane.md index a9a5f768ec8..3a430ad55bd 100644 --- a/doc/user/clusters/crossplane.md +++ b/doc/user/clusters/crossplane.md @@ -151,7 +151,7 @@ kubectl describe globaladdress.compute.gcp.crossplane.io gitlab-ad-globaladdress Resource classes are a way of defining a configuration for the required managed service. We will define the PostgreSQL Resource class -- Define a `gcp-postgres-standard.yaml` resourceclass which contains +- Define a `gcp-postgres-standard.yaml` resource class which contains 1. A default CloudSQLInstanceClass. 1. A CloudSQLInstanceClass with labels. diff --git a/doc/user/clusters/environments.md b/doc/user/clusters/environments.md index a2adf238dda..2b342ceb06d 100644 --- a/doc/user/clusters/environments.md +++ b/doc/user/clusters/environments.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Cluster Environments **(PREMIUM)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13392) for group-level clusters in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/14809) for instance-level clusters in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13392) for group-level clusters in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14809) for instance-level clusters in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. Cluster environments provide a consolidated view of which CI [environments](../../ci/environments/index.md) are deployed to the Kubernetes cluster and it: diff --git a/doc/user/clusters/management_project.md b/doc/user/clusters/management_project.md index 03b4dc45015..c8755af29a3 100644 --- a/doc/user/clusters/management_project.md +++ b/doc/user/clusters/management_project.md @@ -10,7 +10,7 @@ CAUTION: **Warning:** This is an _alpha_ feature, and it is subject to change at any time without prior notice. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32810) in GitLab 12.5 +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32810) in GitLab 12.5 A project can be designated as the management project for a cluster. A management project can be used to run deployment jobs with diff --git a/doc/user/compliance/compliance_dashboard/index.md b/doc/user/compliance/compliance_dashboard/index.md index 25feb6e56bc..08a26a45b17 100644 --- a/doc/user/compliance/compliance_dashboard/index.md +++ b/doc/user/compliance/compliance_dashboard/index.md @@ -1,10 +1,13 @@ --- type: reference, howto +stage: Manage +group: Compliance +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Compliance Dashboard **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36524) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36524) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.8. The Compliance Dashboard gives you the ability to see a group's Merge Request activity by providing a high-level view for all projects in the group. For example, code approved diff --git a/doc/user/compliance/index.md b/doc/user/compliance/index.md index fd4af74e086..d0e73b47358 100644 --- a/doc/user/compliance/index.md +++ b/doc/user/compliance/index.md @@ -1,3 +1,10 @@ +--- +type: reference +stage: Manage +group: Compliance +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Compliance **(ULTIMATE)** The compliance tools provided by GitLab let you keep an eye on various aspects of your project. The diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index cbabed00283..4ceb393af8c 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -1,10 +1,13 @@ --- type: reference, howto +stage: Secure +group: Composition Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # License Compliance **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5483) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5483) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. ## Overview @@ -61,10 +64,14 @@ The following languages and package managers are supported. | Go | [Godep](https://github.com/tools/godep), [go mod](https://github.com/golang/go/wiki/Modules) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Java | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| | .NET | [Nuget](https://www.nuget.org/) (.NET Framework is supported via the [mono project](https://www.mono-project.com/). Windows specific dependencies are not supported at this time.) |[License Finder](https://github.com/pivotal/LicenseFinder)| -| Python | [pip](https://pip.pypa.io/en/stable/) (Python is supported through [requirements.txt](https://pip.readthedocs.io/en/1.1/requirements.html) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock).) |[License Finder](https://github.com/pivotal/LicenseFinder)| +| Python | [pip](https://pip.pypa.io/en/stable/) (Python is supported through [requirements.txt](https://pip.pypa.io/en/1.1/requirements/) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock).) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Ruby | [gem](https://rubygems.org/) |[License Finder](https://github.com/pivotal/LicenseFinder)| | Objective-C, Swift | [Carthage](https://github.com/Carthage/Carthage) |[License Finder](https://github.com/pivotal/LicenseFinder)| +NOTE: **Note:** + +Java 8 and Gradle 1.x projects are not supported. + ### Experimental support The following languages and package managers are [supported experimentally](https://github.com/pivotal/LicenseFinder#experimental-project-types), @@ -135,14 +142,18 @@ License Compliance can be configured using environment variables. | Environment variable | Required | Description | |-----------------------------|----------|-------------| -| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address to download the analyzer from. | | `ADDITIONAL_CA_CERT_BUNDLE` | no | Bundle of trusted CA certificates (currently supported in Pip, Pipenv, Maven, Gradle, Yarn, and NPM projects). | +| `ASDF_JAVA_VERSION` | no | Version of Java to use for the scan. | +| `ASDF_NODEJS_VERSION` | no | Version of Node.js to use for the scan. | +| `ASDF_PYTHON_VERSION` | no | Version of Python to use for the scan. | +| `ASDF_RUBY_VERSION` | no | Version of Ruby to use for the scan. | | `GRADLE_CLI_OPTS` | no | Additional arguments for the gradle executable. If not supplied, defaults to `--exclude-task=test`. | | `LICENSE_FINDER_CLI_OPTS` | no | Additional arguments for the `license_finder` executable. For example, if your project has both Golang and Ruby code stored in different directories and you want to only scan the Ruby code, you can update your `.gitlab-ci-yml` template to specify which project directories to scan, like `LICENSE_FINDER_CLI_OPTS: '--debug --aggregate-paths=. ruby'`. | | `LM_JAVA_VERSION` | no | Version of Java. If set to `11`, Maven and Gradle use Java 11 instead of Java 8. | | `LM_PYTHON_VERSION` | no | Version of Python. If set to `3`, dependencies are installed using Python 3 instead of Python 2.7. | | `MAVEN_CLI_OPTS` | no | Additional arguments for the mvn executable. If not supplied, defaults to `-DskipTests`. | | `PIP_INDEX_URL` | no | Base URL of Python Package Index (default: `https://pypi.org/simple/`). | +| `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address to download the analyzer from. | | `SETUP_CMD` | no | Custom setup for the dependency installation (experimental). | ### Installing custom dependencies @@ -242,7 +253,7 @@ generate a key store file, see the ### Selecting the version of Python > - [Introduced](https://gitlab.com/gitlab-org/security-products/license-management/-/merge_requests/36) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0. -> - In [GitLab 12.2](https://gitlab.com/gitlab-org/gitlab/issues/12032), Python 3.5 became the default. +> - In [GitLab 12.2](https://gitlab.com/gitlab-org/gitlab/-/issues/12032), Python 3.5 became the default. > - In [GitLab 12.7](https://gitlab.com/gitlab-org/security-products/license-management/-/merge_requests/101), Python 3.8 became the default. License Compliance uses Python 3.8 and pip 19.1 by default. @@ -328,7 +339,7 @@ strict-ssl = false ### Configuring Yarn projects -You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc) +You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc/) file. #### Using private Yarn registries @@ -339,7 +350,7 @@ setting to specify its location. For example: -```text +```plaintext npmRegistryServer: "https://npm.example.com" ``` @@ -348,6 +359,137 @@ npmRegistryServer: "https://npm.example.com" You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables). +### Configuring Bower projects + +You can configure Bower projects by using a [`.bowerrc`](https://bower.io/docs/config/#bowerrc-specification) +file. + +#### Using private Bower registries + +If you have a private Bower registry you can use the +[`registry`](https://bower.io/docs/config/#bowerrc-specification) +setting to specify its location. + +For example: + +```plaintext +{ + "registry": "https://registry.bower.io" +} +``` + +#### Custom root certificates for Bower + +You can supply a custom root certificate to complete TLS verification by using the +`ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables), or by +specifying a `ca` setting in a [`.bowerrc`](https://bower.io/docs/config/#bowerrc-specification) +file. + +### Configuring Conan projects + +You can configure [Conan](https://conan.io/) projects by adding a `.conan` directory to your +project root. The project root serves as the [`CONAN_USER_HOME`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-user-home). + +Consult the [Conan](https://docs.conan.io/en/latest/reference/config_files/conan.conf.html#conan-conf) +documentation for a list of settings that you can apply. + +The `license_scanning` job runs in a [Debian 10](https://www.debian.org/releases/buster/) Docker +image. The supplied image ships with some build tools such as [CMake](https://cmake.org/) and [GCC](https://gcc.gnu.org/). +However, not all project types are supported by default. To install additional tools needed to +compile dependencies, use a [`before_script`](../../../ci/yaml/README.md#before_script-and-after_script) +to install the necessary build tools using the [`apt`](https://wiki.debian.org/PackageManagementTools) +package manager. For a comprehensive list, consult [the Conan documentation](https://docs.conan.io/en/latest/introduction.html#all-platforms-all-build-systems-and-compilers). + +The default [Conan](https://conan.io/) configuration sets [`CONAN_LOGIN_USERNAME`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name) +to `ci_user`, and binds [`CONAN_PASSWORD`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-password-conan-password-remote-name) +to the [`CI_JOB_TOKEN`](../../../ci/variables/predefined_variables.md) +for the running job. This allows Conan projects to fetch packages from a [GitLab Conan Repository](../../packages/conan_repository/#fetching-conan-package-information-from-the-gitlab-package-registry) +if a GitLab remote is specified in the `.conan/remotes.json` file. + +To override the default credentials specify a [`CONAN_LOGIN_USERNAME_{REMOTE_NAME}`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name) +matching the name of the remote specified in the `.conan/remotes.json` file. + +NOTE: **Note:** +[MSBuild](https://github.com/mono/msbuild#microsoftbuild-msbuild) projects aren't supported. The +`license_scanning` image ships with [Mono](https://www.mono-project.com/) and [MSBuild](https://github.com/mono/msbuild#microsoftbuild-msbuild). +Additional setup may be required to build packages for this project configuration. + +#### Using private Conan registries + +By default, [Conan](https://conan.io/) uses the `conan-center` remote. For example: + +```json +{ + "remotes": [ + { + "name": "conan-center", + "url": "https://conan.bintray.com", + "verify_ssl": true + } + ] +} +``` + +To fetch dependencies from an alternate remote, specify that remote in a `.conan/remotes.json`. For +example: + +```json +{ + "remotes": [ + { + "name": "gitlab", + "url": "https://gitlab.com/api/v4/packages/conan", + "verify_ssl": true + } + ] +} +``` + +If credentials are required to authenticate then you can configure a [protected variable](../../../ci/variables/README.md#protect-a-custom-variable) +following the naming convention described in the [`CONAN_LOGIN_USERNAME` documentation](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name). + +#### Custom root certificates for Conan + +You can provide custom certificates by adding a `.conan/cacert.pem` file to the project root and +setting [`CA_CERT_PATH`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-cacert-path) +to `.conan/cacert.pem`. + +If you specify the `ADDITIONAL_CA_CERT_BUNDLE` [environment variable](#available-variables), this +variable's X.509 certificates are installed in the Docker image's default trust store and Conan is +configured to use this as the default `CA_CERT_PATH`. + +### Configuring Go projects + +To configure [Go modules](https://github.com/golang/go/wiki/Modules) +based projects, specify [environment variables](https://golang.org/pkg/cmd/go/#hdr-Environment_variables) +in the `license_scanning` job's [variables](#available-variables) section in `.gitlab-ci.yml`. + +If a project has [vendored](https://golang.org/pkg/cmd/go/#hdr-Vendor_Directories) its modules, +then the combination of the `vendor` directory and `mod.sum` file are used to detect the software +licenses associated with the Go module dependencies. + +#### Using private Go registries + +You can use the [`GOPRIVATE`](https://golang.org/pkg/cmd/go/#hdr-Environment_variables) +and [`GOPROXY`](https://golang.org/pkg/cmd/go/#hdr-Environment_variables) +environment variables to control where modules are sourced from. Alternatively, you can use +[`go mod vendor`](https://golang.org/ref/mod#tmp_28) to vendor a project's modules. + +#### Custom root certificates for Go + +You can specify the [`-insecure`](https://golang.org/pkg/cmd/go/internal/get/) flag by exporting the +[`GOFLAGS`](https://golang.org/cmd/go/#hdr-Environment_variables) +environment variable. For example: + +```yaml +include: + - template: License-Scanning.gitlab-ci.yml + +license_scanning: + variables: + GOFLAGS: '-insecure' +``` + ### Migration from `license_management` to `license_scanning` In GitLab 12.8 a new name for `license_management` job was introduced. This change was made to improve clarity around the purpose of the scan, which is to scan and collect the types of licenses present in a projects dependencies. @@ -450,15 +592,21 @@ license_scanning: The License Compliance job should now use local copies of the License Compliance analyzers to scan your code and generate security reports, without requiring internet access. -Additional configuration may be needed for connecting to [private Maven repositories](#using-private-maven-repos), -[private NPM registries](#using-private-npm-registries), [private Yarn registries](#using-private-yarn-registries), and [private Python repositories](#using-private-python-repos). +Additional configuration may be needed for connecting to +[private Bower registries](#using-private-bower-registries), +[private Conan registries](#using-private-bower-registries), +[private Go registries](#using-private-go-registries), +[private Maven repositories](#using-private-maven-repos), +[private NPM registries](#using-private-npm-registries), +[private Python repositories](#using-private-python-repos), +and [private Yarn registries](#using-private-yarn-registries). Exact name matches are required for [project policies](#project-policies-for-license-compliance) when running in an offline environment ([see related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212388)). ## Project policies for License Compliance -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5940) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5940) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.4. From the project's settings: @@ -496,7 +644,7 @@ Searching for Licenses: ## License Compliance report under pipelines -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5491) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5491) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2. From your project's left sidebar, navigate to **CI/CD > Pipelines** and click on the pipeline ID that has a `license_scanning` job to see the Licenses tab with the listed @@ -518,7 +666,7 @@ but commented out to help encourage others to add to it in the future. --> ## License list -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13582) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13582) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.7. The License list allows you to see your project's licenses and key details about them. @@ -554,3 +702,78 @@ Policies can be configured by maintainers of the project. Developers of the project can view the policies configured in a project.  + +## Troubleshooting + +### `ERROR -- : asdf: No preset version installed for command` + +This error occurs when the version of the tools used by your project +do not match the version of the pre-installed tools available in the +`license_scanning` Docker image. The `license_scanning` job uses +[asdf-vm](https://asdf-vm.com/) to activate the appropriate version of +a tool that your project relies on. For example, if your project relies on a specific +version of [Node.js](https://nodejs.org/) or any other supported tool you can +specify the desired version by adding a +[`.tool-versions`](https://asdf-vm.com/#/core-configuration?id=tool-versions) file to the project +or using the appropriate [`ASDF_<tool>_VERSION`](https://asdf-vm.com/#/core-configuration?id=environment-variables) environment variable to +activate the appropriate version. + +For example, the following `.tool-versions` file will activate version `12.16.3` of [Node.js](https://nodejs.org/) +and version `2.6.6` of [Ruby](https://www.ruby-lang.org/). + +```plaintext +nodejs 12.16.3 +ruby 2.6.6 +``` + +The next example shows how to activate the same versions of the tools mentioned above by using environment variables defined in your +project's `.gitlab-ci.yml` file. + +```yaml +include: + - template: License-Scanning.gitlab-ci.yml + +license_scanning: + variables: + ASDF_NODEJS_VERSION: '12.16.3' + ASDF_RUBY_VERSION: '2.6.6' +``` + +A full list of variables can be found in [environment variables](#available-variables). + +To find out what tools are pre-installed in the `license_scanning` Docker image use the following command: + +```shell +$ docker run --entrypoint='' registry.gitlab.com/gitlab-org/security-products/analyzers/license-finder:3 /bin/bash -lc 'asdf list' +golang + 1.14 +gradle + 6.3 +java + adopt-openjdk-11.0.7+10 + adopt-openjdk-8u242-b08 +maven + 3.6.3 +nodejs + 10.20.1 + 12.16.3 +php + 7.4.5 +python + 2.7.18 + 3.8.2 +ruby + 2.6.6 +sbt + 1.3.8 +``` + +To interact with the `license_scanning` runtime environment use the following command: + +```shell +$ docker run -it --entrypoint='' registry.gitlab.com/gitlab-org/security-products/analyzers/license-finder:3 /bin/bash -l +root@6abb70e9f193:~# +``` + +NOTE: **Note:** +Selecting a custom version of [Mono](https://www.mono-project.com/) or [.NET Core](https://dotnet.microsoft.com/download/dotnet-core) is currently not supported. diff --git a/doc/user/discussions/img/add_another_suggestion_to_batch_v13_1.jpg b/doc/user/discussions/img/add_another_suggestion_to_batch_v13_1.jpg Binary files differnew file mode 100644 index 00000000000..e8aa4b7c730 --- /dev/null +++ b/doc/user/discussions/img/add_another_suggestion_to_batch_v13_1.jpg diff --git a/doc/user/discussions/img/add_first_suggestion_to_batch_v13_1.jpg b/doc/user/discussions/img/add_first_suggestion_to_batch_v13_1.jpg Binary files differnew file mode 100644 index 00000000000..d15f5d53e55 --- /dev/null +++ b/doc/user/discussions/img/add_first_suggestion_to_batch_v13_1.jpg diff --git a/doc/user/discussions/img/apply_batch_of_suggestions_v13_1.jpg b/doc/user/discussions/img/apply_batch_of_suggestions_v13_1.jpg Binary files differnew file mode 100644 index 00000000000..3e1e9c20af9 --- /dev/null +++ b/doc/user/discussions/img/apply_batch_of_suggestions_v13_1.jpg diff --git a/doc/user/discussions/img/quickly_assign_commenter_v13_1.png b/doc/user/discussions/img/quickly_assign_commenter_v13_1.png Binary files differnew file mode 100644 index 00000000000..e19a3ed4f75 --- /dev/null +++ b/doc/user/discussions/img/quickly_assign_commenter_v13_1.png diff --git a/doc/user/discussions/img/remove_suggestion_from_batch_v13_1.jpg b/doc/user/discussions/img/remove_suggestion_from_batch_v13_1.jpg Binary files differnew file mode 100644 index 00000000000..fa8331effdb --- /dev/null +++ b/doc/user/discussions/img/remove_suggestion_from_batch_v13_1.jpg diff --git a/doc/user/discussions/img/suggestions_custom_commit_messages_v12_7.png b/doc/user/discussions/img/suggestions_custom_commit_messages_v12_7.png Binary files differdeleted file mode 100644 index 8bbc0fcb99b..00000000000 --- a/doc/user/discussions/img/suggestions_custom_commit_messages_v12_7.png +++ /dev/null diff --git a/doc/user/discussions/img/suggestions_custom_commit_messages_v13_1.jpg b/doc/user/discussions/img/suggestions_custom_commit_messages_v13_1.jpg Binary files differnew file mode 100644 index 00000000000..a4c9df0ebb9 --- /dev/null +++ b/doc/user/discussions/img/suggestions_custom_commit_messages_v13_1.jpg diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 9f24807ddf4..5ee11c553af 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Threads The ability to contribute conversationally is offered throughout GitLab. @@ -48,7 +54,7 @@ to address feedback and lets you hide threads that are no longer relevant. ### Commit threads in the context of a merge request -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/31847) in GitLab 10.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/31847) in GitLab 10.3. For reviewers with commit-based workflow, it may be useful to add threads to specific commit diffs in the context of a merge request. These threads will @@ -290,9 +296,10 @@ edit existing comments. Non-team members are restricted from adding or editing c Additionally, locked issues and merge requests can not be reopened. -## Merge Request Reviews **(PREMIUM)** +## Merge Request Reviews -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4213) in GitLab 11.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4213) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/28154) to GitLab Core in 13.1. When looking at a Merge Request diff, you are able to start a review. This allows you to create comments inside a Merge Request that are **only visible to you** until published, @@ -364,7 +371,7 @@ Replying to this email will, consequentially, create a new comment on the associ ## Filtering notes -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/26723) in GitLab 11.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/26723) in GitLab 11.5. For issues with many comments like activity notes and user comments, sometimes finding useful information can be hard. There is a way to filter comments from single notes and threads for merge requests and issues. @@ -384,7 +391,7 @@ from any device you're logged into. ## Suggest Changes -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/18008) in GitLab 11.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/18008) in GitLab 11.6. As a reviewer, you're able to suggest code changes with a simple Markdown syntax in Merge Request Diff threads. Then, the @@ -402,10 +409,7 @@ the merge request authored by the user that applied them.  -1. Click **Comment**. - - NOTE: **Note:** - If you're using GitLab Premium, GitLab.com Silver, and higher tiers, the thread will display [Review](#merge-request-reviews-premium) options. Click either **Start a review**, **Add comment now**, or **Add to review** to obtain the same result. +1. Click either **Start a review** or **Add to review** to add your comment to a [review](#merge-request-reviews), or **Add comment now** to add the comment to the thread immediately. The Suggestion in the comment can be applied by the merge request author directly from the merge request: @@ -419,7 +423,7 @@ branch. [Developer permission](../permissions.md) is required to do so. ### Multi-line Suggestions -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53310) in GitLab 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53310) in GitLab 11.10. Reviewers can also suggest changes to multiple lines with a single Suggestion within merge request diff threads by adjusting the range offsets. The @@ -451,38 +455,66 @@ instead of the usual three. ### Configure the commit message for applied Suggestions -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13086) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13086) in GitLab 12.7. + +GitLab uses a default commit message +when applying Suggestions: `Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)` -GitLab uses `Apply suggestion to %{file_path}` by default as commit messages -when applying Suggestions. This commit message can be customized to -follow any guidelines you might have. To do so, expand the **Merge requests** +For example, consider that a user applied 3 suggestions to 2 different files, the default commit message will be: **Apply 3 suggestion(s) to 2 file(s)** + +These commit messages can be customized to follow any guidelines you might have. To do so, expand the **Merge requests** tab within your project's **General** settings and change the **Merge suggestions** text: - + You can also use following variables besides static text: | Variable | Description | Output example | |---|---|---| +| `%{branch_name}` | The name of the branch the Suggestion(s) was(were) applied to. | `my-feature-branch` | +| `%{files_count}` | The number of file(s) to which Suggestion(s) was(were) applied.| **2** | +| `%{file_paths}` | The path(s) of the file(s) Suggestion(s) was(were) applied to. Paths are separated by commas.| `docs/index.md, docs/about.md` | | `%{project_path}` | The project path. | `my-group/my-project` | | `%{project_name}` | The human-readable name of the project. | **My Project** | -| `%{file_path}` | The path of the file the Suggestion is applied to. | `docs/index.md` | -| `%{branch_name}` | The name of the branch the Suggestion is applied on. | `my-feature-branch` | -| `%{username}` | The username of the user applying the Suggestion. | `user_1` | -| `%{user_full_name}` | The full name of the user applying the Suggestion. | **User 1** | +| `%{suggestions_count}` | The number of Suggestions applied.| **3** | +| `%{username}` | The username of the user applying Suggestion(s). | `user_1` | +| `%{user_full_name}` | The full name of the user applying Suggestion(s). | **User 1** | For example, to customize the commit message to output **Addresses user_1's review**, set the custom text to `Addresses %{username}'s review`. NOTE: **Note:** -Custom commit messages for each applied Suggestion will be +Custom commit messages for each applied Suggestion (and for batch Suggestions) will be introduced by [#25381](https://gitlab.com/gitlab-org/gitlab/issues/25381). +### Batch Suggestions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25486) in GitLab 13.1. + +You can apply multiple suggestions at once to reduce the number of commits added +to your branch to address your reviewers' requests. + +1. To start a batch of suggestions that will be applied with a single commit, click **Add suggestion to batch**: + +  + +1. Add as many additional suggestions to the batch as you wish: + +  + +1. To remove suggestions, click **Remove from batch**: + +  + +1. Having added all the suggestions to your liking, when ready, click **Apply suggestions**: + +  + ## Start a thread by replying to a standard comment -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/30299) in GitLab 11.9 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30299) in GitLab 11.9 To reply to a standard (non-thread) comment, you can use the **Reply to comment** button. @@ -500,3 +532,15 @@ to the original comment, so a note about when it was last edited will appear und This feature only exists for Issues, Merge requests, and Epics. Commits, Snippets and Merge request diff threads are not supported yet. + +## Assign an issue to the commenting user + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/191455) in GitLab 13.1. + +You can assign an issue to a user who made a comment. + +In the comment, click the **More Actions** menu and click **Assign to commenting user**. + +Click the button again to unassign the commenter. + + diff --git a/doc/user/feature_highlight.md b/doc/user/feature_highlight.md index 47f8671afae..9b52c178493 100644 --- a/doc/user/feature_highlight.md +++ b/doc/user/feature_highlight.md @@ -3,11 +3,11 @@ > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/16379) in GitLab 10.5 Feature highlights are represented by a pulsing blue dot. Hovering over the dot -will open up callout with more information. +will display more information. They are used to emphasize a certain feature and make something more visible to the user. You can dismiss any feature highlight permanently by clicking the "Got it" link -at the bottom of the callout. There isn't a way to restore the feature highlight +at the bottom of the modal window. There isn't a way to restore the feature highlight after it has been dismissed.  diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index d615b67f3d2..6522f5c4053 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -31,6 +31,10 @@ gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAA GitLab.com sends emails from the `mg.gitlab.com` domain via [Mailgun](https://www.mailgun.com/) and has its own dedicated IP address (`198.61.254.240`). +## Backups + +[See our backup strategy](https://about.gitlab.com/handbook/engineering/infrastructure/production/#backups). + ## Alternative SSH port GitLab.com can be reached via a [different SSH port](https://about.gitlab.com/blog/2016/02/18/gitlab-dot-com-now-supports-an-alternate-git-plus-ssh-port/) for `git+ssh`. @@ -78,6 +82,7 @@ Below are the current settings regarding [GitLab CI/CD](../../ci/README.md). | Scheduled Pipeline Cron | `*/5 * * * *` | `19 * * * *` | | [Max jobs in active pipelines](../../administration/instance_limits.md#number-of-jobs-in-active-pipelines) | `500` for Free tier, unlimited otherwise | Unlimited | [Max pipeline schedules in projects](../../administration/instance_limits.md#number-of-pipeline-schedules) | `10` for Free tier, `50` for all paid tiers | Unlimited | +| [Max number of instance level variables](../../administration/instance_limits.md#number-of-instance-level-variables) | `25` | `25` | ## Repository size limit @@ -136,7 +141,7 @@ The `gitlab-shared-runners-manager-X.gitlab.com` fleet of Runners are dedicated Jobs handled by the shared Runners on GitLab.com (`shared-runners-manager-X.gitlab.com`), **will be timed out after 3 hours**, regardless of the timeout configured in a -project. Check the issues [4010](https://gitlab.com/gitlab-com/infrastructure/issues/4010) and [4070](https://gitlab.com/gitlab-com/infrastructure/issues/4070) for the reference. +project. Check the issues [4010](https://gitlab.com/gitlab-com/infrastructure/-/issues/4010) and [4070](https://gitlab.com/gitlab-com/infrastructure/-/issues/4070) for the reference. Below are the shared Runners settings. @@ -152,7 +157,7 @@ Below are the shared Runners settings. Linux Shared Runners on GitLab.com provide a way to run commands in a CI job before the Runner attempts to run `git init` and `git fetch` to download a GitLab repository. The -[pre_clone_script](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) +[`pre_clone_script`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) can be used for: - Seeding the build directory with repository data @@ -219,7 +224,7 @@ sentry_dsn = "X" "google-tags=gitlab-com,srm", "google-use-internal-ip", "google-zone=us-east1-d", - "engine-opt=mtu=1460", # Set MTU for container interface, for more information check https://gitlab.com/gitlab-org/gitlab-runner/issues/3214#note_82892928 + "engine-opt=mtu=1460", # Set MTU for container interface, for more information check https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3214#note_82892928 "google-machine-image=PROJECT/global/images/IMAGE", "engine-opt=ipv6", # This will create IPv6 interfaces in the containers. "engine-opt=fixed-cidr-v6=fc00::/7", @@ -243,7 +248,7 @@ During the beta period, the [shared runner pipeline quota](../admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota-starter-only) will apply for groups and projects in the same way as Linux Runners. This may change when the beta period ends, as discussed in this -[related issue](https://gitlab.com/gitlab-org/gitlab/issues/30834). +[related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/30834). Windows Shared Runners on GitLab.com automatically autoscale by launching virtual machines on the Google Cloud Platform. This solution uses @@ -367,7 +372,7 @@ test: release we will update the autoscaler to enable the pre-provisioning of virtual machines. This will significantly reduce the time it takes to provision a VM on the Windows fleet. You can - follow along in the [related issue](https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/autoscaler/issues/32). + follow along in the [related issue](https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/autoscaler/-/issues/32). - The Windows Shared Runner fleet may be unavailable occasionally for maintenance or updates. - The Windows Shared Runner virtual machine instances do not use the @@ -415,44 +420,44 @@ different database servers. The list of GitLab.com specific settings (and their defaults) is as follows: -| Setting | GitLab.com | Default | -|:------------------------------------|:--------------------------------------------------------------------|:--------------------------------------| -| archive_command | `/usr/bin/envdir /etc/wal-e.d/env /opt/wal-e/bin/wal-e wal-push %p` | empty | -| archive_mode | on | off | -| autovacuum_analyze_scale_factor | 0.01 | 0.01 | -| autovacuum_max_workers | 6 | 3 | -| autovacuum_vacuum_cost_limit | 1000 | -1 | -| autovacuum_vacuum_scale_factor | 0.01 | 0.02 | -| checkpoint_completion_target | 0.7 | 0.9 | -| checkpoint_segments | 32 | 10 | -| effective_cache_size | 338688MB | Based on how much memory is available | -| hot_standby | on | off | -| hot_standby_feedback | on | off | -| log_autovacuum_min_duration | 0 | -1 | -| log_checkpoints | on | off | -| log_line_prefix | `%t [%p]: [%l-1]` | empty | -| log_min_duration_statement | 1000 | -1 | -| log_temp_files | 0 | -1 | -| maintenance_work_mem | 2048MB | 16 MB | -| max_replication_slots | 5 | 0 | -| max_wal_senders | 32 | 0 | -| max_wal_size | 5GB | 1GB | -| shared_buffers | 112896MB | Based on how much memory is available | -| shared_preload_libraries | pg_stat_statements | empty | -| shmall | 30146560 | Based on the server's capabilities | -| shmmax | 123480309760 | Based on the server's capabilities | -| wal_buffers | 16MB | -1 | -| wal_keep_segments | 512 | 10 | -| wal_level | replica | minimal | -| statement_timeout | 15s | 60s | -| idle_in_transaction_session_timeout | 60s | 60s | +| Setting | GitLab.com | Default | +|:--------------------------------------|:--------------------------------------------------------------------|:--------------------------------------| +| `archive_command` | `/usr/bin/envdir /etc/wal-e.d/env /opt/wal-e/bin/wal-e wal-push %p` | empty | +| `archive_mode` | on | off | +| `autovacuum_analyze_scale_factor` | 0.01 | 0.01 | +| `autovacuum_max_workers` | 6 | 3 | +| `autovacuum_vacuum_cost_limit` | 1000 | -1 | +| `autovacuum_vacuum_scale_factor` | 0.01 | 0.02 | +| `checkpoint_completion_target` | 0.7 | 0.9 | +| `checkpoint_segments` | 32 | 10 | +| `effective_cache_size` | 338688MB | Based on how much memory is available | +| `hot_standby` | on | off | +| `hot_standby_feedback` | on | off | +| `log_autovacuum_min_duration` | 0 | -1 | +| `log_checkpoints` | on | off | +| `log_line_prefix` | `%t [%p]: [%l-1]` | empty | +| `log_min_duration_statement` | 1000 | -1 | +| `log_temp_files` | 0 | -1 | +| `maintenance_work_mem` | 2048MB | 16 MB | +| `max_replication_slots` | 5 | 0 | +| `max_wal_senders` | 32 | 0 | +| `max_wal_size` | 5GB | 1GB | +| `shared_buffers` | 112896MB | Based on how much memory is available | +| `shared_preload_libraries` | pg_stat_statements | empty | +| `shmall` | 30146560 | Based on the server's capabilities | +| `shmmax` | 123480309760 | Based on the server's capabilities | +| `wal_buffers` | 16MB | -1 | +| `wal_keep_segments` | 512 | 10 | +| `wal_level` | replica | minimal | +| `statement_timeout` | 15s | 60s | +| `idle_in_transaction_session_timeout` | 60s | 60s | Some of these settings are in the process being adjusted. For example, the value for `shared_buffers` is quite high and as such we are looking into adjusting it. More information on this particular change can be found at -<https://gitlab.com/gitlab-com/infrastructure/issues/1555>. An up to date list +<https://gitlab.com/gitlab-com/infrastructure/-/issues/1555>. An up to date list of proposed changes can be found at -<https://gitlab.com/gitlab-com/infrastructure/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=database&label_name[]=change>. +<https://gitlab.com/gitlab-com/infrastructure/-/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=database&label_name[]=change>. ## Unicorn @@ -553,7 +558,7 @@ GitLab.com: On GitLab.com, projects, groups, and snippets created As of GitLab 12.2 (July 2019), projects, groups, and snippets have the -[**Internal** visibility](../../public_access/public_access.md#internal-projects) setting [disabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/issues/12388). +[**Internal** visibility](../../public_access/public_access.md#internal-projects) setting [disabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/12388). ### SSH maximum number of connections @@ -563,6 +568,10 @@ If more than the maximum number of allowed connections occur concurrently, they dropped and users get [an `ssh_exchange_identification` error](../../topics/git/troubleshooting_git.md#ssh_exchange_identification-error). +### Import/export + +To help avoid abuse, project and group imports, exports, and export downloads are rate limited. See [Project import/export rate limits](../../user/project/settings/import_export.md#rate-limits) and [Group import/export rate limits](../../user/group/settings/import_export.md#rate-limits) for details. + ## GitLab.com Logging We use [Fluentd](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#fluentd) to parse our logs. Fluentd sends our logs to diff --git a/doc/user/group/bulk_editing/index.md b/doc/user/group/bulk_editing/index.md index a90097c848f..c4ccc1f7b52 100644 --- a/doc/user/group/bulk_editing/index.md +++ b/doc/user/group/bulk_editing/index.md @@ -1,31 +1,74 @@ -# Bulk editing issues, merge requests, and epics at the group level **(PREMIUM)** +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7249) for issues in [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12719) for merge requests in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7250) for epics in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. +# Bulk editing issues, epics, and merge requests at the group level **(PREMIUM)** -## Editing milestones and labels +NOTE: **Note:** +Bulk editing issues and merge requests is also available at the **project level**. +For more details, see [Bulk editing issues, epics, and merge requests](../../project/bulk_editing.md). -> **Notes:** -> -> - A permission level of `Reporter` or higher is required in order to manage issues. -> - A permission level of `Developer` or higher is required in order to manage merge requests. -> - A permission level of `Reporter` or higher is required in order to manage epics. +If you want to update attributes across multiple issues, epics, or merge requests in a group, you +can do it by bulk editing them, that is, editing them together. -By using the bulk editing feature: + -- Milestones can be updated simultaneously across multiple issues or merge requests. -- Labels can be updated simultaneously across multiple issues, merge requests, or epics. +## Bulk edit issues at the group level - +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7249) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. + +NOTE: **Note:** +You need a permission level of [Reporter or higher](../../permissions.md) to manage issues. + +When bulk editing issues in a group, you can edit the following attributes: + +- Milestone +- Labels + +To update multiple project issues at the same time: + +1. In a group, go to **{issues}** **Issues > List**. +1. Click **Edit issues**. A sidebar on the right-hand side of your screen appears with editable fields. +1. Select the checkboxes next to each issue you want to edit. +1. Select the appropriate fields and their values from the sidebar. +1. Click **Update all**. + +## Bulk edit epics + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7250) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. + +NOTE: **Note:** +You need a permission level of [Reporter or higher](../../permissions.md) to manage epics. + +When bulk editing epics in a group, you can edit their labels. + +To update multiple epics at the same time: + +1. In a group, go to **{epic}** **Epics > List**. +1. Click **Edit epics**. A sidebar on the right-hand side of your screen appears with editable fields. +1. Check the checkboxes next to each epic you want to edit. +1. Select the appropriate fields and their values from the sidebar. +1. Click **Update all**. + +## Bulk edit merge requests at the group level + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12719) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. + +NOTE: **Note:** +You need a permission level of [Developer or higher](../../permissions.md) to manage merge requests. + +When bulk editing merge requests in a group, you can edit the following attributes: + +- Milestone +- Labels -To bulk update group issues, merge requests, or epics: +To update multiple group merge requests at the same time: -1. Navigate to the issues, merge requests, or epics list. -1. Click **Edit issues**, **Edit merge requests**, or **Edit epics**. - - This will open a sidebar on the right-hand side where editable fields - for milestones and labels will be displayed. - - Checkboxes will also appear beside each issue, merge request, or epic. -1. Check the checkbox beside each issue, merge request, or epic to be edited. -1. Select the desired new values from the sidebar. +1. In a group, go to **{merge-request}** **Merge Requests**. +1. Click **Edit merge requests**. A sidebar on the right-hand side of your screen appears with + editable fields. +1. Select the checkboxes next to each merge request you want to edit. +1. Select the appropriate fields and their values from the sidebar. 1. Click **Update all**. diff --git a/doc/user/group/clusters/index.md b/doc/user/group/clusters/index.md index 5e6ff980c8b..5cdac7ae892 100644 --- a/doc/user/group/clusters/index.md +++ b/doc/user/group/clusters/index.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Group-level Kubernetes clusters -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/34758) in GitLab 11.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34758) in GitLab 11.6. Similar to [project-level](../../project/clusters/index.md) and [instance-level](../../instance/clusters/index.md) Kubernetes clusters, @@ -23,8 +23,8 @@ and troubleshooting applications for your group cluster, see ## RBAC compatibility -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/29398) in GitLab 11.4. -> - [Project namespace restriction](https://gitlab.com/gitlab-org/gitlab-foss/issues/51716) was introduced in GitLab 11.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29398) in GitLab 11.4. +> - [Project namespace restriction](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51716) was introduced in GitLab 11.5. For each project under a group with a Kubernetes cluster, GitLab creates a restricted service account with [`edit` privileges](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) @@ -65,7 +65,7 @@ for deployments with a cluster not managed by GitLab, you must ensure: - The project's deployment service account has permissions to deploy to [`KUBE_NAMESPACE`](../../project/clusters/index.md#deployment-variables). - `KUBECONFIG` correctly reflects any changes to `KUBE_NAMESPACE` - (this is [not automatic](https://gitlab.com/gitlab-org/gitlab/issues/31519)). Editing + (this is [not automatic](https://gitlab.com/gitlab-org/gitlab/-/issues/31519)). Editing `KUBE_NAMESPACE` directly is discouraged. NOTE: **Note:** @@ -74,7 +74,7 @@ the resources required to run them even if you choose to manage your own cluster ### Clearing the cluster cache -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31759) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31759) in GitLab 12.6. If you choose to allow GitLab to manage your cluster for you, GitLab stores a cached version of the namespaces and service accounts it creates for your projects. If you diff --git a/doc/user/group/contribution_analytics/img/group_stats_table.png b/doc/user/group/contribution_analytics/img/group_stats_table.png Binary files differindex f1d1031fa18..1f58b9717d0 100644 --- a/doc/user/group/contribution_analytics/img/group_stats_table.png +++ b/doc/user/group/contribution_analytics/img/group_stats_table.png diff --git a/doc/user/group/contribution_analytics/index.md b/doc/user/group/contribution_analytics/index.md index 03f0ad6ad1c..bcc6d958427 100644 --- a/doc/user/group/contribution_analytics/index.md +++ b/doc/user/group/contribution_analytics/index.md @@ -2,13 +2,12 @@ type: reference stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Contribution Analytics **(STARTER)** > - Introduced in [GitLab Starter](https://about.gitlab.com/pricing/) 8.3. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3090) for subgroups in GitLab 12.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3090) for subgroups in GitLab 12.2. ## Overview @@ -63,7 +62,7 @@ Contributions per group member are also presented in tabular format. Click a col - Number of opened issues - Number of closed issues - Number of opened MRs -- Number of accepted MRs +- Number of merged MRs - Number of total contributions  diff --git a/doc/user/group/custom_project_templates.md b/doc/user/group/custom_project_templates.md index e47a281141d..ebeacda24c6 100644 --- a/doc/user/group/custom_project_templates.md +++ b/doc/user/group/custom_project_templates.md @@ -1,10 +1,13 @@ --- type: reference +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Custom group-level project templates **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6861) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6861) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.6. When you create a new [project](../project/index.md), creating it based on custom project templates is a convenient option. diff --git a/doc/user/group/epics/img/bulk_editing.png b/doc/user/group/epics/img/bulk_editing.png Binary files differdeleted file mode 100644 index 85610bef83e..00000000000 --- a/doc/user/group/epics/img/bulk_editing.png +++ /dev/null diff --git a/doc/user/group/epics/img/issue_list_v13_1.png b/doc/user/group/epics/img/issue_list_v13_1.png Binary files differnew file mode 100644 index 00000000000..f08f774e986 --- /dev/null +++ b/doc/user/group/epics/img/issue_list_v13_1.png diff --git a/doc/user/group/epics/index.md b/doc/user/group/epics/index.md index d53acaf502a..85164292265 100644 --- a/doc/user/group/epics/index.md +++ b/doc/user/group/epics/index.md @@ -1,11 +1,14 @@ --- type: reference, howto +stage: Plan +group: Portfolio Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Epics **(PREMIUM)** > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.2. -> - Single-level Epics [were moved](https://gitlab.com/gitlab-org/gitlab/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. +> - Single-level Epics [were moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. Epics let you manage your portfolio of projects more efficiently and with less effort by tracking groups of issues that share a theme, across projects and @@ -25,7 +28,7 @@ milestones. To learn what you can do with an epic, see [Manage epics](manage_epics.md). Possible actions include: - [Create an epic](manage_epics.md#create-an-epic) -- [Bulk-edit epics](manage_epics.md#bulk-edit-epics) +- [Bulk-edit epics](../bulk_editing/index.md#bulk-edit-epics) - [Delete an epic](manage_epics.md#delete-an-epic) - [Close an epic](manage_epics.md#close-an-epic) - [Reopen a closed epic](manage_epics.md#reopen-a-closed-epic) @@ -67,7 +70,7 @@ This feature comes with a feature flag enabled by default. For steps to disable ## Multi-level child epics **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8333) in GitLab Ultimate 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8333) in GitLab Ultimate 11.7. Any epic that belongs to a group, or subgroup of the parent epic's group, is eligible to be added. New child epics appear at the top of the list of epics in the **Epics and Issues** tab. @@ -86,11 +89,11 @@ To set a **Start date** and **Due date** for an epic, select one of the followin - **Fixed**: Enter a fixed value. - **From milestones**: Inherit a dynamic value from the milestones currently assigned to the epic's issues. Note that GitLab 12.5 replaced this option with **Inherited**. -- **Inherited**: Inherit a dynamic value from the epic's issues, child epics, and milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7332) in GitLab 12.5 to replace **From milestones**). +- **Inherited**: Inherit a dynamic value from the epic's issues, child epics, and milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7332) in GitLab 12.5 to replace **From milestones**). ### From milestones -> [Replaced](https://gitlab.com/gitlab-org/gitlab/issues/7332) in GitLab 12.5 by **Inherited**. +> [Replaced](https://gitlab.com/gitlab-org/gitlab/-/issues/7332) in GitLab 12.5 by **Inherited**. If you select **From milestones** for the start date, GitLab will automatically set the date to be earliest start date across all milestones that are currently assigned to the issues that are added to the epic. @@ -105,7 +108,7 @@ These are dynamic dates which are recalculated if any of the following occur: ### Inherited -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7332) in GitLab 12.5 to replace **From milestones**. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7332) in GitLab 12.5 to replace **From milestones**. If you select: @@ -127,7 +130,7 @@ then the parent epic's start date will reflect the change and this will propagat ## Roadmap in epics -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7327) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7327) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.10. If your epic contains one or more [child epics](#multi-level-child-epics-ultimate) which have a [start or due date](#start-date-and-due-date), a diff --git a/doc/user/group/epics/manage_epics.md b/doc/user/group/epics/manage_epics.md index 50eb0c64f4b..26d5cb51e01 100644 --- a/doc/user/group/epics/manage_epics.md +++ b/doc/user/group/epics/manage_epics.md @@ -1,8 +1,11 @@ --- type: howto +stage: Plan +group: Portfolio Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -<!-- When adding a new section here, remember to mention it in index.md#manage-epics --> +<!-- When adding a new h2 section here, remember to mention it in index.md#manage-epics --> # Manage epics **(PREMIUM)** @@ -30,7 +33,8 @@ You will be taken to the new epic where can edit the following details: An epic's page contains the following tabs: -- **Epics and Issues**: epics and issues added to this epic. Child epics, and their issues, are shown in a tree view. +- **Epics and Issues**: epics and issues added to this epic. Child epics, and their issues, are + shown in a tree view. - Click the <kbd>></kbd> beside a parent epic to reveal the child epics and issues. - Hover over the total counts to see a breakdown of open and closed items. - **Roadmap**: a roadmap view of child epics which have start and due dates. @@ -39,17 +43,8 @@ An epic's page contains the following tabs: ## Bulk-edit epics -You can edit multiple epics at once. For example, to apply labels to multiple epics: - -1. Go to the Epics list. -1. Click **Edit epics**. - - Checkboxes appear next to each epic. - - A sidebar on the right-hand side appears with an editable field for labels. -1. Select the checkbox next to each epic to be edited. -1. Select the labels you want. -1. Click **Update all**. - - +You can edit multiple epics at once. To learn how to do it, visit +[Bulk editing issues, epics, and merge requests at the group level](../bulk_editing/index.md#bulk-edit-epics). ## Delete an epic @@ -92,7 +87,7 @@ link in the issue sidebar. ## Search for an epic from epics list page > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.5. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/37081) to the [Premium](https://about.gitlab.com/pricing/) tier in GitLab 12.8. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) to the [Premium](https://about.gitlab.com/pricing/) tier in GitLab 12.8. You can search for an epic from the list of epics using filtered search bar (similar to that of Issues and Merge Requests) based on following parameters: @@ -134,8 +129,8 @@ confidential** checkbox. ### Enable Confidential Epics **(PREMIUM ONLY)** -The Confidential Epics feature is under development and not ready for production use. It's deployed behind a -feature flag that is **disabled by default**. +The Confidential Epics feature is under development and not ready for production use. +It's deployed behind a feature flag that is **disabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) can enable it for your instance. @@ -175,14 +170,14 @@ To add an issue to an epic: 1. Identify the issue to be added, using either of the following methods: - Paste the link of the issue. - Search for the desired issue by entering part of the issue's title, then selecting the desired - match (introduced in [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/9126)). + match (introduced in [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/-/issues/9126)). If there are multiple issues to be added, press <kbd>Spacebar</kbd> and repeat this step. 1. Click **Add**. #### Create an issue from an epic -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5419) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5419) in GitLab 12.7. Creating an issue from an epic enables you to maintain focus on the broader context of the epic while dividing work into smaller parts. @@ -195,46 +190,49 @@ To create an issue from an epic: 1. From the **Project** dropdown, select the project in which the issue should be created. 1. Click **Create issue**. +### Remove an issue from an epic + +You can remove issues from an epic when you're on the epic's details page. +After you remove an issue from an epic, the issue will no longer be associated with this epic. + To remove an issue from an epic: -1. Click the <kbd>x</kbd> button in the epic's issue list. -1. Click **Remove** in the **Remove issue** warning message. +1. Click the **Remove** (**{close}**) button next to the issue you want to remove. + The **Remove issue** warning appears. +1. Click **Remove**. + + ### Reorder issues assigned to an epic -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9367) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9367) in GitLab 12.5. -New issues are added to the top of their list in the **Epics and Issues** tab. -You can reorder the list of issues. Issues and child epics cannot be intermingled. +New issues appear at the top of the list in the **Epics and Issues** tab. +You can reorder the list of issues by dragging them. To reorder issues assigned to an epic: 1. Go to the **Epics and Issues** tab. -1. Drag and drop issues into the desired order. - -To reorder child epics assigned to an epic: - -1. Go to the **Epics and Issues** tab. -1. Drag and drop epics into the desired order. +1. Drag issues into the desired order. ### Move issues between epics **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. -New issues are added to the top of their list in the **Epics and Issues** -tab. You can move issues from one epic to another. Issues and child epics cannot be intermingled. +New issues appear at the top of the list in the **Epics and Issues** +tab. You can move issues from one epic to another. To move an issue to another epic: 1. Go to the **Epics and Issues** tab. -1. Drag and drop issues into the desired parent epic. +1. Drag issues into the desired parent epic. ### Promote an issue to an epic -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3777) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.6. -> - In [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/37081), it was moved to the Premium tier. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3777) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) to [GitLab Premium](https://about.gitlab.com/pricing/) in 12.8. -If you have [permissions](../../permissions.md) to close an issue and create an +If you have the necessary [permissions](../../permissions.md) to close an issue and create an epic in the parent group, you can promote an issue to an epic with the `/promote` [quick action](../../project/quick_actions.md#quick-actions-for-issues-merge-requests-and-epics). Only issues from projects that are in groups can be promoted. When attempting to promote a confidential @@ -263,7 +261,8 @@ To add a child epic to an epic: 1. Click **Add an epic**. 1. Identify the epic to be added, using either of the following methods: - Paste the link of the epic. - - Search for the desired issue by entering part of the epic's title, then selecting the desired match (introduced in [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/issues/9126)). + - Search for the desired issue by entering part of the epic's title, then selecting the desired + match (introduced in [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/-/issues/9126)). If there are multiple epics to be added, press <kbd>Spacebar</kbd> and repeat this step. 1. Click **Add**. @@ -272,7 +271,7 @@ To add a child epic to an epic: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.0. -New child epics are added to the top of their list in the **Epics and Issues** tab. +New child epics appear at the top of the list in the **Epics and Issues** tab. You can move child epics from one epic to another. When you add an epic that's already linked to a parent epic, the link to its current parent is removed. Issues and child epics cannot be intermingled. @@ -280,19 +279,19 @@ Issues and child epics cannot be intermingled. To move child epics to another epic: 1. Go to the **Epics and Issues** tab. -1. Drag and drop epics into the desired parent epic. +1. Drag epics into the desired parent epic. ### Reorder child epics assigned to an epic -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9367) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9367) in GitLab 12.5. -New child epics are added to the top of their list in the **Epics and Issues** tab. -You can reorder the list of child epics. Issues and child epics cannot be intermingled. +New child epics appear at the top of the list in the **Epics and Issues** tab. +You can reorder the list of child epics. To reorder child epics assigned to an epic: 1. Go to the **Epics and Issues** tab. -1. Drag and drop epics into the desired order. +1. Drag epics into the desired order. ### Remove a child epic from a parent epic diff --git a/doc/user/group/img/ldap_sync_cn_v13_1.png b/doc/user/group/img/ldap_sync_cn_v13_1.png Binary files differnew file mode 100644 index 00000000000..1b0a20b7e64 --- /dev/null +++ b/doc/user/group/img/ldap_sync_cn_v13_1.png diff --git a/doc/user/group/img/ldap_sync_filter_v13_1.png b/doc/user/group/img/ldap_sync_filter_v13_1.png Binary files differnew file mode 100644 index 00000000000..3fc1f28becd --- /dev/null +++ b/doc/user/group/img/ldap_sync_filter_v13_1.png diff --git a/doc/user/group/img/manual_permissions_v13_1.png b/doc/user/group/img/manual_permissions_v13_1.png Binary files differnew file mode 100644 index 00000000000..504e6ab3a42 --- /dev/null +++ b/doc/user/group/img/manual_permissions_v13_1.png diff --git a/doc/user/group/index.md b/doc/user/group/index.md index f36f3b3fd4f..324c912b2be 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Groups @@ -17,7 +20,7 @@ Find your groups by clicking **Groups > Your Groups** in the top navigation.  -> The **Groups** dropdown in the top navigation was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/36234) in [GitLab 11.1](https://about.gitlab.com/releases/2018/07/22/gitlab-11-1-released/#groups-dropdown-in-navigation). +> The **Groups** dropdown in the top navigation was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/36234) in [GitLab 11.1](https://about.gitlab.com/releases/2018/07/22/gitlab-11-1-released/#groups-dropdown-in-navigation). The **Groups** page displays: @@ -183,7 +186,7 @@ of a group: ## Changing the default branch protection of a group -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7583) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7583) in GitLab 12.9. By default, every group inherits the branch protection set at the global level. @@ -214,7 +217,7 @@ There are two different ways to add a new project to a group: ### Default project-creation level -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2534) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2534) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.5. > - Brought to [GitLab Starter](https://about.gitlab.com/pricing/) in 10.7. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25975) to [GitLab Core](https://about.gitlab.com/pricing/) in 11.10. @@ -287,7 +290,7 @@ Alternatively, you can [lock the sharing with group feature](#share-with-group-l ## Sharing a group with another group -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/18328) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18328) in GitLab 12.7. Similarly to [sharing a project with a group](#sharing-a-project-with-a-group), you can share a group with another group to give direct group members access @@ -306,8 +309,50 @@ All the members of the 'Engineering' group will have been added to 'Frontend'. ## Manage group memberships via LDAP -In GitLab Enterprise Edition, it is possible to manage GitLab group memberships using LDAP groups. -See [the GitLab Enterprise Edition documentation](../../integration/ldap.md) for more information. +Group syncing allows LDAP groups to be mapped to GitLab groups. This provides more control over per-group user management. To configure group syncing edit the `group_base` **DN** (`'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org'`). This **OU** contains all groups that will be associated with GitLab groups. + +Group links can be created using either a CN or a filter. These group links are created on the **Group Settings -> LDAP Synchronization** page. After configuring the link, it may take over an hour for the users to sync with the GitLab group. + +For more information on the administration of LDAP and group sync, refer to the [main LDAP documentation](../../administration/auth/ldap/index.md#group-sync-starter-only). + +NOTE: **Note:** +If an LDAP user is a group member when LDAP Synchronization is added, and they are not part of the LDAP group, they will be removed from the group. + +### Creating group links via CN **(STARTER ONLY)** + +To create group links via CN: + +1. Select the **LDAP Server** for the link. +1. Select `LDAP Group cn` as the **Sync method**. +1. In the **LDAP Group cn** text input box, begin typing the CN of the group. There will be a dropdown menu with matching CNs within the configured `group_base`. Select your CN from this list. +1. In the **LDAP Access** section, select the [permission level](../permissions.md) for users synced in this group. +1. Click the `Add Synchronization` button to save this group link. + + + +### Creating group links via filter **(PREMIUM ONLY)** + +To create group links via filter: + +1. Select the **LDAP Server** for the link. +1. Select `LDAP user filter` as the **Sync method**. +1. Input your filter in the **LDAP User filter** box. Follow the [documentation on user filters](../../administration/auth/ldap/index.md#set-up-ldap-user-filter-core-only). +1. In the **LDAP Access** section, select the [permission level](../permissions.md) for users synced in this group. +1. Click the `Add Synchronization` button to save this group link. + + + +### Overriding user permissions **(STARTER ONLY)** + +Since GitLab [v8.15](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/822) LDAP user permissions can now be manually overridden by an admin user. To override a user's permissions: + +1. Go to your group's **Members** page. +1. Select the pencil icon in the row for the user you are editing. +1. Select the orange `Change permissions` button. + + + +Now you will be able to edit the user's permissions from the **Members** page. ## Epics **(ULTIMATE)** @@ -407,11 +452,11 @@ To remove a group and its contents: This action either: - Removes the group, and also queues a background job to delete all projects in that group. -- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/premium/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). +- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium or Silver](https://about.gitlab.com/pricing/premium/) or higher tiers, marks a group for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). ### Restore a group **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/33257) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33257) in GitLab 12.8. To restore a group that is marked for deletion: @@ -460,9 +505,10 @@ This will disable the option for all users who previously had permissions to operate project memberships, so no new users can be added. Furthermore, any request to add a new user to a project through API will not be possible. -#### IP access restriction **(ULTIMATE)** +#### IP access restriction **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1985) in [GitLab Ultimate and Gold](https://about.gitlab.com/pricing/) 12.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1985) in [GitLab Ultimate and Gold](https://about.gitlab.com/pricing/) 12.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/215410) to [GitLab Premium and Silver](https://about.gitlab.com/pricing/) in 13.1. To make sure only people from within your organization can access particular resources, you have the option to restrict access to groups and their @@ -470,28 +516,27 @@ underlying projects, issues, etc, by IP address. This can help ensure that particular content doesn't leave the premises, while not blocking off access to the entire instance. -Add one or more whitelisted IP subnets using CIDR notation in comma separated format to the group settings and anyone +Add one or more allowed IP subnets using CIDR notation in comma separated format to the group settings and anyone coming from a different IP address won't be able to access the restricted content. Restriction currently applies to: - UI. -- [From GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/issues/12874), API access. -- [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/32113), Git actions via SSH. +- [From GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/12874), API access. +- [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/32113), Git actions via SSH. To avoid accidental lock-out, admins and group owners are able to access the group regardless of the IP restriction. #### Allowed domain restriction **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7297) in [GitLab Premium and Silver](https://about.gitlab.com/pricing/) 12.2. +>- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7297) in [GitLab Premium and Silver](https://about.gitlab.com/pricing/) 12.2. +>- Support for specifying multiple email domains [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33143) in GitLab 13.1 -You can restrict access to groups by -allowing only users with email addresses in particular domains to be added to the group. +You can restrict access to groups by allowing only users with email addresses in particular domains to be added to the group. -Add email domains you want to whitelist and users with emails from different -domains won't be allowed to be added to this group. +Add email domains you want to allow and users with emails from different domains won't be allowed to be added to this group. Some domains cannot be restricted. These are the most popular public email domains, such as: @@ -509,7 +554,7 @@ Some domains cannot be restricted. These are the most popular public email domai To enable this feature: 1. Navigate to the group's **Settings > General** page. -1. Expand the **Permissions, LFS, 2FA** section, and enter the domain name into **Restrict membership by email** field. +1. Expand the **Permissions, LFS, 2FA** section, and enter the domain names into **Restrict membership by email** field. You can enter multiple domains by separating each domain with a comma (,). 1. Click **Save changes**. This will enable the domain-checking for all new users added to the group from this moment on. @@ -545,7 +590,7 @@ Define project templates at a group level by setting a group as the template sou #### Disabling email notifications -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/23585) in GitLab 12.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23585) in GitLab 12.2. You can disable all email notifications related to the group, which includes its subgroups and projects. @@ -557,7 +602,7 @@ To enable this feature: #### Disabling group mentions -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21301) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21301) in GitLab 12.6. You can prevent users from being added to a conversation and getting notified when anyone mentions a group in which those users are members. @@ -598,7 +643,7 @@ If your namespace shows `N/A` as the total storage usage, you can trigger a reca #### Group push rules **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34370) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34370) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.8. Group push rules allow group maintainers to set [push rules](../../push_rules/push_rules.md) for newly created projects within the specific group. diff --git a/doc/user/group/insights/index.md b/doc/user/group/insights/index.md index cffbc013e66..dd1f8914392 100644 --- a/doc/user/group/insights/index.md +++ b/doc/user/group/insights/index.md @@ -2,8 +2,7 @@ type: reference, howto stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Insights **(ULTIMATE)** diff --git a/doc/user/group/issues_analytics/img/issues_table_v13_1.png b/doc/user/group/issues_analytics/img/issues_table_v13_1.png Binary files differnew file mode 100644 index 00000000000..0f94f1ba2c5 --- /dev/null +++ b/doc/user/group/issues_analytics/img/issues_table_v13_1.png diff --git a/doc/user/group/issues_analytics/index.md b/doc/user/group/issues_analytics/index.md index df96f2626e1..01cee3d09b8 100644 --- a/doc/user/group/issues_analytics/index.md +++ b/doc/user/group/issues_analytics/index.md @@ -2,14 +2,13 @@ type: reference stage: Manage group: Analytics -To determine the technical writer assigned to the Stage/Group associated with this page, see: - https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Issues Analytics **(PREMIUM)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7478) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.5. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/196561) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9 at the project level. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7478) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196561) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9 at the project level. Issues Analytics is a bar graph which illustrates the number of issues created each month. The default timespan is 13 months, which includes the current month, and the 12 months @@ -35,6 +34,17 @@ shows a total of 15 months for the chart in the GitLab.org group.  +## Drill into the information + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196547) in GitLab 13.1. + +You can examine details of individual issues by browsing the table +located below the chart. + +The chart displays the top 100 issues based on the global page filters. + + + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md new file mode 100644 index 00000000000..2704147dcdd --- /dev/null +++ b/doc/user/group/iterations/index.md @@ -0,0 +1,85 @@ +--- +type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Iterations **(STARTER)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214713) in [GitLab Starter](https://about.gitlab.com/pricing/) 13.1. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's able to be enabled or disabled per-group +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-iterations-core-only). **(CORE ONLY)** + +Iterations are a way to track issues over a period of time. This allows teams +to track velocity and volatility metrics. Iterations can be used with [milestones](../../project/milestones/index.md) +for tracking over different time periods. + +For example, you can use: + +- Milestones for Program Increments, which span 8-12 weeks. +- Iterations for Sprints, which span 2 weeks. + +In GitLab, iterations are similar to milestones, with a few differences: + +- Iterations are only available to groups. +- A group can only have one active iteration at a time. +- Iterations require both a start and an end date. +- Iteration date ranges cannot overlap. + +## View the iterations list + +To view the iterations list, in a group, go to **{issues}** **Issues > Iterations**. +From there you can create a new iteration or click an iteration to get a more detailed view. + +## Create an iteration + +NOTE: **Note:** +A permission level of [Developer or higher](../../permissions.md) is required to create iterations. + +To create an iteration: + +1. In a group, go to **{issues}** **Issues > Iterations**. +1. Click **New iteration**. +1. Enter the title, a description (optional), a start date, and a due date. +1. Click **Create iteration**. The iteration details page opens. + +### Enable Iterations **(CORE ONLY)** + +GitLab Iterations feature is under development and not ready for production use. +It is deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. `:group_iterations` can be enabled or disabled per-group. + +To enable it: + +```ruby +# Instance-wide +Feature.enable(:group_iterations) +# or by group +Feature.enable(:group_iterations, Group.find(<group id>)) +``` + +To disable it: + +```ruby +# Instance-wide +Feature.disable(:group_iterations) +# or by group +Feature.disable(:group_iterations, Group.find(<group id>)) +``` + +<!-- ## Troubleshooting + +Include any troubleshooting steps that you can foresee. If you know beforehand what issues +one might have when setting this up, or when something is changed, or on upgrading, it's +important to describe those, too. Think of things that may go wrong and include them here. +This is important to minimize requests for support, and to avoid doc comments with +questions that you know someone might ask. + +Each scenario can be a third-level heading, e.g. `### Getting error message X`. +If you have none to add when creating a doc, leave this section in place +but commented out to help encourage others to add to it in the future. --> diff --git a/doc/user/group/roadmap/index.md b/doc/user/group/roadmap/index.md index 6bee552d433..614ed700cfc 100644 --- a/doc/user/group/roadmap/index.md +++ b/doc/user/group/roadmap/index.md @@ -1,14 +1,17 @@ --- type: reference +stage: Plan +group: Portfolio Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Roadmap **(PREMIUM)** > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.5. -> - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/198062), Roadmaps were moved to the Premium tier. -> - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/5164) and later, the epic bars show epics' title, progress, and completed weight percentage. -> - Milestones appear in Roadmaps in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/6802), and later. -> - Feature flag removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29641). +> - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/198062), Roadmaps were moved to the Premium tier. +> - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/5164) and later, the epic bars show epics' title, progress, and completed weight percentage. +> - Milestones appear in roadmaps in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/6802), and later. +> - Feature flag for milestones visible in roadmaps removed in [GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29641). Epics and milestones within a group containing **Start date** and/or **Due date** can be visualized in a form of a timeline (that is, a Gantt chart). The Roadmap page @@ -45,7 +48,7 @@ Roadmaps can also be [visualized inside an epic](../epics/index.md#roadmap-in-ep ## Timeline duration > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. -> - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/198062), Timelines were moved to the Premium tier. +> - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/198062), Timelines were moved to the Premium tier. Roadmap supports the following date ranges: diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index a3d9a14df10..81684038dc2 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -1,5 +1,8 @@ --- type: reference, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # SAML SSO for GitLab.com groups **(SILVER ONLY)** @@ -54,14 +57,14 @@ We recommend setting the NameID format to `Persistent` unless using a field (suc ### SSO enforcement -- [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5291) in GitLab 11.8. -- [Improved](https://gitlab.com/gitlab-org/gitlab/issues/9255) in GitLab 11.11 with ongoing enforcement in the GitLab UI. +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5291) in GitLab 11.8. +- [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9255) in GitLab 11.11 with ongoing enforcement in the GitLab UI. With this option enabled, users must use your group's GitLab single sign on URL to be added to the group or be added via SCIM. Users cannot be added manually, and may only access project/group resources via the UI by signing in through the SSO URL. However, users will not be prompted to log via SSO on each visit. GitLab will check whether a user has authenticated through the SSO link, and will only prompt the user to login via SSO if the session has expired. -We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab/issues/9152) in the future. +We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab/-/issues/9152) in the future. When SSO enforcement is enabled for a group, users cannot share a project in the group outside the top-level group, even if the project is forked. @@ -82,7 +85,7 @@ When this option is enabled: Upon successful authentication, GitLab prompts the user with options, based on the email address received from the configured identity provider: - To create a unique account with the newly received email address. -- If the received email address matches one of the user's verified GitLab email addresses, the option to convert the existing account to a group-managed account. ([Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/issues/13481).) +- If the received email address matches one of the user's verified GitLab email addresses, the option to convert the existing account to a group-managed account. ([Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/13481).) Since use of the group-managed account requires the use of SSO, users of group-managed accounts will lose access to these accounts when they are no longer able to authenticate with the connected identity provider. In the case of an offboarded employee who has been removed from your identity provider: @@ -100,7 +103,7 @@ Feature.enable(:group_managed_accounts) ##### Credentials inventory for Group-managed accounts **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/38133) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38133) in GitLab 12.8. Owners who manage user accounts in a group can view the following details of personal access tokens and SSH keys: @@ -140,7 +143,7 @@ Once a lifetime for personal access tokens is set, GitLab will: ##### Outer forks restriction for Group-managed accounts -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34648) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34648) in GitLab 12.9. Groups with group-managed accounts can disallow forking of projects to destinations outside the group. To do so, enable the "Prohibit outer forks" option in **Settings > SAML SSO**. @@ -148,7 +151,7 @@ When enabled, projects within the group can only be forked to other destinations ##### Other restrictions for Group-managed accounts -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12420) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12420) in GitLab 12.9. Projects within groups with enabled group-managed accounts are not to be shared with: @@ -231,7 +234,7 @@ NOTE: **Note:** GitLab is unable to provide support for IdPs that are not listed |----------|---------------| | ADFS (Active Directory Federation Services) | [Create a Relying Party Trust](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) | | Azure | [Configuring single sign-on to applications](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/configure-single-sign-on-non-gallery-applications) | -| Okta | [Setting up a SAML application in Okta](https://developer.okta.com/docs/guides/saml-application-setup/overview/) | +| Okta | [Setting up a SAML application in Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/overview/) | | OneLogin | [Use the OneLogin SAML Test Connector](https://onelogin.service-now.com/support?id=kb_article&sys_id=93f95543db109700d5505eea4b96198f) | When [configuring your identify provider](#configuring-your-identity-provider), please consider the notes below for specific providers to help avoid common issues and as a guide for terminology used. @@ -373,8 +376,8 @@ To proceed with configuring Group SAML SSO instead, you'll need to enable the `g Group SAML on a self-managed instance is limited when compared to the recommended [instance-wide SAML](../../../integration/saml.md). The recommended solution allows you to take advantage of: -- [LDAP compatibility](../../../administration/auth/ldap.md). -- [LDAP group Sync](../../../administration/auth/how_to_configure_ldap_gitlab_ee/index.md#group-sync). +- [LDAP compatibility](../../../administration/auth/ldap/index.md). +- [LDAP Group Sync](../index.md#manage-group-memberships-via-ldap) - [Required groups](../../../integration/saml.md#required-groups-starter-only). - [Admin groups](../../../integration/saml.md#admin-groups-starter-only). - [Auditor groups](../../../integration/saml.md#auditor-groups-starter-only). diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index f66c8a788b6..a891962b38e 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -1,5 +1,8 @@ --- type: howto, reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # SCIM provisioning using SAML SSO for GitLab.com groups **(SILVER ONLY)** @@ -117,7 +120,7 @@ Once synchronized, changing the field mapped to `id` and `externalId` will likel ### Okta configuration steps -The SAML application that was created during [Single sign-on](index.md#okta-setup-notes) setup for [Okta](https://developer.okta.com/docs/guides/saml-application-setup/overview/) now needs to be set up for SCIM. +The SAML application that was created during [Single sign-on](index.md#okta-setup-notes) setup for [Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/overview/) now needs to be set up for SCIM. Before proceeding, be sure to complete the [GitLab configuration](#gitlab-configuration) process. 1. Sign in to Okta. diff --git a/doc/user/group/settings/img/export_panel.png b/doc/user/group/settings/img/export_panel.png Binary files differdeleted file mode 100644 index 2a987f04e35..00000000000 --- a/doc/user/group/settings/img/export_panel.png +++ /dev/null diff --git a/doc/user/group/settings/img/export_panel_v13_0.png b/doc/user/group/settings/img/export_panel_v13_0.png Binary files differnew file mode 100644 index 00000000000..36549e1f3f5 --- /dev/null +++ b/doc/user/group/settings/img/export_panel_v13_0.png diff --git a/doc/user/group/settings/img/import_panel_v13_1.png b/doc/user/group/settings/img/import_panel_v13_1.png Binary files differnew file mode 100644 index 00000000000..ce2eb579446 --- /dev/null +++ b/doc/user/group/settings/img/import_panel_v13_1.png diff --git a/doc/user/group/settings/img/new_group_navigation_v13_1.png b/doc/user/group/settings/img/new_group_navigation_v13_1.png Binary files differnew file mode 100644 index 00000000000..98d45a694b6 --- /dev/null +++ b/doc/user/group/settings/img/new_group_navigation_v13_1.png diff --git a/doc/user/group/settings/import_export.md b/doc/user/group/settings/import_export.md index 3f14fea673b..cca918d44f3 100644 --- a/doc/user/group/settings/import_export.md +++ b/doc/user/group/settings/import_export.md @@ -1,3 +1,9 @@ +--- +type: reference +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- # Group Import/Export > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2888) in GitLab 13.0 as an experimental feature. May change in future releases. @@ -61,7 +67,7 @@ For more details on the specific data persisted in a group export, see the 1. In the **Advanced** section, click the **Export Group** button. -  +  1. Once the export is generated, you should receive an e-mail with a link to the [exported contents](#exported-contents) in a compressed tar archive, with contents in JSON format. @@ -69,12 +75,37 @@ For more details on the specific data persisted in a group export, see the 1. Alternatively, you can come back to the project settings and download the file from there by clicking **Download export**, or generate a new file by clicking **Regenerate export**. +NOTE: **Note:** +The maximum import file size can be set by the Administrator, default is 50MB. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin UI](../../admin_area/settings/account_and_limit_settings.md). + ### Between CE and EE You can export groups from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) and vice versa. If you're exporting a group from the Enterprise Edition to the Community Edition, you may lose data that is retained only in the Enterprise Edition. For more information, see [downgrading from EE to CE](../../../README.md). +## Importing the group + +1. Navigate to the New Group page, either via the `+` button in the top navigation bar, or the **New subgroup** button +on an existing group's page. + +  + +1. On the New Group page, select the **Import group** tab. + +  + +1. Enter your group name. + +1. Accept or modify the associated group URL. + +1. Click **Choose file** + +1. Select the file that you exported in the [exporting a group](#exporting-a-group) section. + +1. Click **Import group** to begin importing. Your newly imported group page will appear shortly. + ## Version history GitLab can import bundles that were exported from a different GitLab deployment. @@ -92,7 +123,8 @@ For example: To help avoid abuse, users are rate limited to: -| Request Type | Limit | -| ---------------- | ------------------------------ | -| Export | 1 group every 5 minutes | -| Download export | 10 downloads every 10 minutes | +| Request Type | Limit | +| ---------------- | ---------------------------------------- | +| Export | 30 groups every 5 minutes | +| Download export | 10 downloads per group every 10 minutes | +| Import | 30 groups every 5 minutes | diff --git a/doc/user/group/subgroups/index.md b/doc/user/group/subgroups/index.md index 49f6ddc3986..842e2082be4 100644 --- a/doc/user/group/subgroups/index.md +++ b/doc/user/group/subgroups/index.md @@ -4,7 +4,7 @@ type: reference, howto, concepts # Subgroups -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/2772) in GitLab 9.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/2772) in GitLab 9.0. GitLab supports up to 20 levels of subgroups, also known as nested groups or hierarchical groups. levels of groups. @@ -139,7 +139,7 @@ From the image above, we can deduce the following things: - Administrator is the Owner and member of **all** subgroups and for that reason, as with User3, there is no indication of an ancestor group. -[From](https://gitlab.com/gitlab-org/gitlab/issues/21727) GitLab 12.6, you can filter +[From](https://gitlab.com/gitlab-org/gitlab/-/issues/21727) GitLab 12.6, you can filter this list using dropdown on the right side:  diff --git a/doc/user/incident_management/index.md b/doc/user/incident_management/index.md index 73d6e0e055d..48805bda909 100644 --- a/doc/user/incident_management/index.md +++ b/doc/user/incident_management/index.md @@ -7,47 +7,49 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Incident Management GitLab offers solutions for handling incidents in your applications and services, -from setting up an alert with Prometheus, to receiving a notification through a -monitoring tool like Slack, and [setting up Zoom calls](#zoom-integration-in-issues) with your -support team. Incidents can display [metrics](#embed-metrics-in-incidents-and-issues) -and [logs](#view-logs-from-metrics-panel). +such as setting up Prometheus alerts, displaying metrics, and sending notifications. ## Configure incidents **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4925) in GitLab Ultimate 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in GitLab Ultimate 11.11. -You can enable or disable Incident Management features in your project's -**{settings}** **Settings > Operations > Incidents**. Issues can be created for -each alert triggered, and separate email notifications can be sent to users with -[Developer permissions](../permissions.md). Appropriately configured alerts include an -[embedded chart](../project/integrations/prometheus.md#embedding-metrics-based-on-alerts-in-incident-issues) -for the query corresponding to the alert. You can also configure GitLab to -[close issues](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate) -when you receive notification that the alert is resolved. - - +You can enable or disable Incident Management features in the GitLab user interface +to create issues when alerts are triggered: -### Create issues from alerts +1. Navigate to **{settings}** **Settings > Operations > Incidents** and expand + **Incidents**: -You can create GitLab issues from an alert notification. These issues contain -information about the alerts to help you diagnose the source of the alerts. +  -1. Visit your project's **{settings}** **Settings > Operations > Incidents**. -1. Select the **Create an issue** checkbox for GitLab to create an issue from - the incident. -1. Select the template from the **Issue Template** dropdown. - You can create your own [issue templates](../project/description_templates.md#creating-issue-templates) - to [use within Incident Management](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate). +1. For GitLab versions 11.11 and greater, you can select the **Create an issue** + checkbox to create an issue based on your own + [issue templates](../project/description_templates.md#creating-issue-templates). + For more information, see + [Taking Action on Incidents](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate) **(ULTIMATE)**. +1. To create issues from alerts, select the template in the **Issue Template** + select box. +1. To send [separate email notifications](#notify-developers-of-alerts) to users + with [Developer permissions](../permissions.md), select + **Send a separate email notification to Developers**. 1. Click **Save changes**. -## Notify developers of alerts +Appropriately configured alerts include an +[embedded chart](../project/integrations/prometheus.md#embedding-metrics-based-on-alerts-in-incident-issues) +for the query corresponding to the alert. You can also configure GitLab to +[close issues](../project/integrations/prometheus.md#taking-action-on-incidents-ultimate) +when you receive notification that the alert is resolved. + +### Notify developers of alerts GitLab can react to the alerts triggered from your applications and services -by creating issues and alerting developers through email. GitLab sends these emails -to [owners and maintainers](../permissions.md) of the project. They contain details -of the alert, and a link for more information. +by creating issues and alerting developers through email. By default, GitLab +sends these emails to [owners and maintainers](../permissions.md) of the project. +These emails contain details of the alert, and a link for more information. + +To send separate email notifications to users with +[Developer permissions](../permissions.md), see [Configure incidents](#configure-incidents-ultimate). -### Configure Prometheus alerts +## Configure Prometheus alerts You can set up Prometheus alerts in: @@ -57,7 +59,7 @@ You can set up Prometheus alerts in: Prometheus alerts are created by the special Alert Bot user. You can't remove this user, but it does not count toward your license limit. -### Configure external generic alerts +## Configure external generic alerts GitLab can accept alerts from any source through a generic webhook receiver. When [configuring the generic alerts integration](../project/integrations/generic_alerts.md), @@ -65,7 +67,7 @@ GitLab creates a unique endpoint which receives a JSON-formatted, customizable p ## Embed metrics in incidents and issues -You can embed metrics anywhere GitLab Markdown is used, such as descriptions, +You can embed metrics anywhere [GitLab Markdown](../markdown.md) is used, such as descriptions, comments on issues, and merge requests. Embedding metrics helps you share them when discussing incidents or performance issues. You can output the dashboard directly into any issue, merge request, epic, or any other Markdown text field in GitLab @@ -78,10 +80,9 @@ in incidents and issue templates. ### Context menu -From each of the embedded metrics panels, you can access more details -about the data you're viewing from a context menu. You can access the context menu -by clicking the **{ellipsis_v}** **More actions** dropdown box above the -upper right corner of the panel. The options are: +You can view more details about an embedded metrics panel from the context menu. +To access the context menu, click the **{ellipsis_v}** **More actions** dropdown box +above the upper right corner of the panel. The options are: - [View logs](#view-logs-from-metrics-panel). - **Download CSV** - Data from embedded charts can be @@ -89,7 +90,7 @@ upper right corner of the panel. The options are: #### View logs from metrics panel -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201846) in GitLab Ultimate 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201846) in GitLab Ultimate 12.8. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. Viewing logs from a metrics panel can be useful if you're triaging an application @@ -97,16 +98,16 @@ incident and need to [explore logs](../project/integrations/prometheus.md#view-l from across your application. These logs help you understand what is affecting your application's performance and resolve any problems. -## Slack integration +## Integrate incidents with Slack Slack slash commands allow you to control GitLab and view GitLab content without leaving Slack. Learn how to [set up Slack slash commands](../project/integrations/slack_slash_commands.md) and how to [use the available slash commands](../../integration/slash_commands.md). -## Zoom integration in issues +## Integrate issues with Zoom GitLab enables you to [associate a Zoom meeting with an issue](../project/issues/associate_zoom_meeting.md) for synchronous communication during incident management. After starting a Zoom -call for an incident, you can associate the conference call with an issue, so your -team members can join without requesting a link. +call for an incident, you can associate the conference call with an issue. Your +team members can join the Zoom call without requesting a link. diff --git a/doc/user/infrastructure/index.md b/doc/user/infrastructure/index.md index 65237bf24e0..c17d1831b50 100644 --- a/doc/user/infrastructure/index.md +++ b/doc/user/infrastructure/index.md @@ -22,136 +22,146 @@ Amazon S3 or Google Cloud Storage. Its features include: - Locking and unlocking state. - Remote Terraform plan and apply execution. -To get started, there are two different options when using GitLab managed Terraform State. +To get started with a GitLab-managed Terraform State, there are two different options: -- Use a local machine -- Use GitLab CI +- [Use a local machine](#get-started-using-local-development). +- [Use GitLab CI](#get-started-using-gitlab-ci). -## Get Started using local development +## Get started using local development -If you are planning to only run `terraform plan` and `terraform apply` commands from your local machine, this is a simple way to get started. +If you plan to only run `terraform plan` and `terraform apply` commands from your +local machine, this is a simple way to get started: -First, create your project on your GitLab instance. +1. Create your project on your GitLab instance. +1. Navigate to **{settings}** **Settings > General** and note your **Project name** + and **Project ID**. +1. Define the Terraform backend in your Terraform project to be: -Next, define the Terraform backend in your Terraform project to be: - -```hcl -terraform { - backend "http" { - } -} -``` - -Finally, you need to run `terraform init` on your local machine and pass in the following options. The below example is using GitLab.com: - -```bash -terraform init \ - -backend-config="address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>" \ - -backend-config="lock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ - -backend-config="unlock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ - -backend-config="username=<YOUR-USERNAME>" \ - -backend-config="password=<YOUR-ACCESS-TOKEN>" \ - -backend-config="lock_method=POST" \ - -backend-config="unlock_method=DELETE" \ - -backend-config="retry_wait_min=5" -``` - -This will initialize your Terraform state and store that state within your GitLab project. - -NOTE: YOUR-PROJECT-ID and YOUR-PROJECT-NAME can be accessed from the project main page. - -## Get Started using a GitLab CI - -Another route is to leverage GitLab CI to run your `terraform plan` and `terraform apply` commands. - -### Configure the CI variables + ```hcl + terraform { + backend "http" { + } + } + ``` -To use the Terraform backend, [first create a Personal Access Token](../profile/personal_access_tokens.md) with the `api` scope. Keep in mind that the Terraform backend is restricted to tokens with [Maintainer access](../permissions.md) to the repository. +1. Create a [Personal Access Token](../profile/personal_access_tokens.md) with + the `api` scope. The Terraform backend is restricted to users with + [Maintainer access](../permissions.md) to the repository. + +1. On your local machine, run `terraform init`, passing in the following options, + replacing `<YOUR-PROJECT-NAME>`, `<YOUR-PROJECT-ID>`, `<YOUR-USERNAME>` and + `<YOUR-ACCESS-TOKEN>` with the relevant values. This command initializes your + Terraform state, and stores that state within your GitLab project. This example + uses `gitlab.com`: + + ```shell + terraform init \ + -backend-config="address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>" \ + -backend-config="lock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ + -backend-config="unlock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-PROJECT-NAME>/lock" \ + -backend-config="username=<YOUR-USERNAME>" \ + -backend-config="password=<YOUR-ACCESS-TOKEN>" \ + -backend-config="lock_method=POST" \ + -backend-config="unlock_method=DELETE" \ + -backend-config="retry_wait_min=5" + ``` -To keep the Personal Access Token secure, add it as a [CI/CD environment variable](../../ci/variables/README.md). In this example we set ours to the ENV: `GITLAB_TF_PASSWORD`. +Next, [configure the backend](#configure-the-backend). -If you are planning to use the ENV on a branch which is not protected, make sure to set the variable protection settings correctly. +## Get started using GitLab CI -### Configure the Terraform backend +If you don't want to start with local development, you can also use GitLab CI to +run your `terraform plan` and `terraform apply` commands. -Next we need to define the [http backend](https://www.terraform.io/docs/backends/types/http.html). In your Terraform project add the following code block in a `.tf` file such as `backend.tf` or wherever you desire to define the remote backend: +Next, [configure the backend](#configure-the-backend). -```hcl -terraform { - backend "http" { - } -} -``` +## Configure the backend -### Configure the CI YAML file +After executing the `terraform init` command, you must configure the Terraform backend +and the CI YAML file: -Finally, configure a `.gitlab-ci.yaml`, which lives in the root of your project repository. +CAUTION: **Important:** +The Terraform backend is restricted to users with [Maintainer access](../permissions.md) +to the repository. -In our case we are using a pre-built image: +1. In your Terraform project, define the [HTTP backend](https://www.terraform.io/docs/backends/types/http.html) + by adding the following code block in a `.tf` file (such as `backend.tf`) to + define the remote backend: -```yaml -image: - name: hashicorp/terraform:light - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' -``` - -We then define some environment variables to make life easier. `GITLAB_TF_ADDRESS` is the URL of the GitLab instance where this pipeline runs, and `TF_ROOT` is the directory where the Terraform commands must be executed. - -```yaml -variables: - GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} - TF_ROOT: ${CI_PROJECT_DIR}/environments/cloudflare/production - -cache: - paths: - - .terraform -``` + ```hcl + terraform { + backend "http" { + } + } + ``` -In a `before_script`, pass a `terraform init` call containing configuration parameters. -These parameters correspond to variables required by the -[http backend](https://www.terraform.io/docs/backends/types/http.html): +1. In the root directory of your project repository, configure a `.gitlab-ci.yaml` file. + This example uses a pre-built image: -```yaml -before_script: - - cd ${TF_ROOT} - - terraform --version - - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=${GITLAB_USER_LOGIN}" -backend-config="password=${GITLAB_TF_PASSWORD}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" + ```yaml + image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + ``` -stages: - - validate - - build - - test - - deploy +1. In the `.gitlab-ci.yaml` file, define some environment variables to ease development. In this + example, `GITLAB_TF_ADDRESS` is the URL of the GitLab instance where this pipeline + runs, and `TF_ROOT` is the directory where the Terraform commands must be executed: -validate: - stage: validate - script: - - terraform validate + ```yaml + variables: + GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} + TF_ROOT: ${CI_PROJECT_DIR}/environments/cloudflare/production -plan: - stage: build - script: - - terraform plan - - terraform show + cache: + paths: + - .terraform + ``` -apply: - stage: deploy - environment: - name: production - script: - - terraform apply - dependencies: - - plan - when: manual - only: - - master -``` +1. In a `before_script`, pass a `terraform init` call containing configuration parameters + corresponding to variables required by the + [HTTP backend](https://www.terraform.io/docs/backends/types/http.html): -### Push to GitLab + ```yaml + before_script: + - cd ${TF_ROOT} + - terraform --version + - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=gitlab-ci-token" -backend-config="password=${CI_JOB_TOKEN}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" + + stages: + - validate + - build + - test + - deploy + + validate: + stage: validate + script: + - terraform validate + + plan: + stage: build + script: + - terraform plan + - terraform show + + apply: + stage: deploy + environment: + name: production + script: + - terraform apply + dependencies: + - plan + when: manual + only: + - master + ``` -Pushing your project to GitLab triggers a CI job pipeline, which runs the `terraform init`, `terraform validate`, and `terraform plan` commands automatically. +1. Push your project to GitLab, which triggers a CI job pipeline. This pipeline runs + the `terraform init`, `terraform validate`, and `terraform plan` commands. The output from the above `terraform` commands should be viewable in the job logs. @@ -161,14 +171,14 @@ See [this reference project](https://gitlab.com/nicholasklick/gitlab-terraform-a ## Output Terraform Plan information into a merge request -Using the [GitLab Terraform Report Artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform), +Using the [GitLab Terraform Report artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform), you can expose details from `terraform plan` runs directly into a merge request widget, enabling you to see statistics about the resources that Terraform will create, modify, or destroy. -Let's explore how to configure a GitLab Terraform Report Artifact: +Let's explore how to configure a GitLab Terraform Report artifact: -1. First, for simplicity, let's define a few reusable variables to allow us to +1. For simplicity, let's define a few reusable variables to allow us to refer to these files multiple times: ```yaml @@ -177,96 +187,39 @@ Let's explore how to configure a GitLab Terraform Report Artifact: PLAN_JSON: tfplan.json ``` -1. Next we need to install `jq`, a [lightweight and flexible command-line JSON processor](https://stedolan.github.io/jq/). We will also create an alias for a specific `jq` command that parses out the extact information we want to extract from the `terraform plan` output: - -```yaml -before_script: - - apk --no-cache add jq - - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" -``` - -1. Finally, we define a `script` that runs `terraform plan` and also a `terraform show` which pipes the output and converts the relevant bits into a store variable `PLAN_JSON`. This json is then leveraged to create a [GitLab Terraform Report Artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform). - -The terraform report obtains a Terraform tfplan.json file. The collected Terraform plan report will be uploaded to GitLab as an artifact and will be automatically shown in merge requests. - -```yaml -plan: - stage: build - script: - - terraform plan -out=$PLAN - - terraform show --json $PLAN | convert_report > $PLAN_JSON - artifacts: - name: plan - paths: - - $PLAN - reports: - terraform: $PLAN_JSON -``` - -A full `.gitlab-ci.yaml` file could look like this: - -```yaml -image: - name: hashicorp/terraform:light - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - -# Default output file for Terraform plan -variables: - GITLAB_TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PROJECT_NAME} - PLAN: plan.tfplan - PLAN_JSON: tfplan.json - TF_ROOT: ${CI_PROJECT_DIR} - -cache: - paths: - - .terraform - -before_script: - - apk --no-cache add jq - - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - - cd ${TF_ROOT} - - terraform --version - - terraform init -backend-config="address=${GITLAB_TF_ADDRESS}" -backend-config="lock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="unlock_address=${GITLAB_TF_ADDRESS}/lock" -backend-config="username=${GITLAB_USER_LOGIN}" -backend-config="password=${GITLAB_TF_PASSWORD}" -backend-config="lock_method=POST" -backend-config="unlock_method=DELETE" -backend-config="retry_wait_min=5" - -stages: - - validate - - build - - deploy +1. Install `jq`, a + [lightweight and flexible command-line JSON processor](https://stedolan.github.io/jq/). +1. Create an alias for a specific `jq` command that parses out the information we + want to extract from the `terraform plan` output: -validate: - stage: validate - script: - - terraform validate + ```yaml + before_script: + - apk --no-cache add jq + - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" + ``` -plan: - stage: build - script: - - terraform plan -out=$PLAN - - terraform show --json $PLAN | convert_report > $PLAN_JSON - artifacts: - name: plan - paths: - - ${TF_ROOT}/plan.tfplan - reports: - terraform: ${TF_ROOT}/tfplan.json +1. Define a `script` that runs `terraform plan` and `terraform show`. These commands + pipe the output and convert the relevant bits into a store variable `PLAN_JSON`. + This JSON is used to create a + [GitLab Terraform Report artifact](../../ci/pipelines/job_artifacts.md#artifactsreportsterraform). + The Terraform report obtains a Terraform `tfplan.json` file. The collected + Terraform plan report is uploaded to GitLab as an artifact, and is shown in merge requests. -# Separate apply job for manual launching Terraform as it can be destructive -# action. -apply: - stage: deploy - environment: - name: production - script: - - terraform apply -input=false $PLAN - dependencies: - - plan - when: manual - only: - - master + ```yaml + plan: + stage: build + script: + - terraform plan -out=$PLAN + - terraform show --json $PLAN | convert_report > $PLAN_JSON + artifacts: + name: plan + paths: + - $PLAN + reports: + terraform: $PLAN_JSON + ``` -``` + For a full example, see [Example `.gitlab-ci.yaml` file](#example-gitlab-ciyaml-file). 1. Running the pipeline displays the widget in the merge request, like this: diff --git a/doc/user/instance/clusters/index.md b/doc/user/instance/clusters/index.md index 495bfdeed6e..8059b8ca642 100644 --- a/doc/user/instance/clusters/index.md +++ b/doc/user/instance/clusters/index.md @@ -1,6 +1,6 @@ # Instance-level Kubernetes clusters -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/39840) in GitLab 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39840) in GitLab 11.11. ## Overview @@ -12,8 +12,7 @@ projects. ## Cluster precedence -GitLab will try [to match](../../../ci/environments/index.md#scoping-environments-with-specs) clusters in -the following order: +GitLab will try to match clusters in the following order: - Project-level clusters. - Group-level clusters. diff --git a/doc/user/instance_statistics/dev_ops_score.md b/doc/user/instance_statistics/dev_ops_score.md index 216a79f8beb..73b9532015d 100644 --- a/doc/user/instance_statistics/dev_ops_score.md +++ b/doc/user/instance_statistics/dev_ops_score.md @@ -1,7 +1,7 @@ # DevOps Score -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/30469) in GitLab 9.3. -> - [Renamed from Conversational Development Index](https://gitlab.com/gitlab-org/gitlab/issues/20976) in GitLab 12.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30469) in GitLab 9.3. +> - [Renamed from Conversational Development Index](https://gitlab.com/gitlab-org/gitlab/-/issues/20976) in GitLab 12.6. NOTE: **Note:** Your GitLab instance's [usage ping](../admin_area/settings/usage_statistics.md#usage-ping-core-only) must be activated in order to use this feature. diff --git a/doc/user/instance_statistics/index.md b/doc/user/instance_statistics/index.md index 15e061b118e..b09651e04ee 100644 --- a/doc/user/instance_statistics/index.md +++ b/doc/user/instance_statistics/index.md @@ -1,6 +1,6 @@ # Instance statistics -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41416) in GitLab 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41416) in GitLab 11.2. Instance statistics gives users or admins access to instance-wide analytics. They are accessible to all users by default (GitLab admins can restrict its diff --git a/doc/user/instance_statistics/user_cohorts.md b/doc/user/instance_statistics/user_cohorts.md index b3ef99473e4..8da2d57d474 100644 --- a/doc/user/instance_statistics/user_cohorts.md +++ b/doc/user/instance_statistics/user_cohorts.md @@ -1,6 +1,6 @@ # Cohorts -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/23361) in GitLab 9.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/23361) in GitLab 9.1. As a benefit of having the [usage ping active](../admin_area/settings/usage_statistics.md), GitLab lets you analyze the users' activities over time of your GitLab installation. @@ -24,6 +24,6 @@ How do we measure the activity of users? GitLab considers a user active if: - The user signs in. - The user has Git activity (whether push or pull). -- The user visits pages related to Dashboards, Projects, Issues, and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/54947) in GitLab 11.8). +- The user visits pages related to Dashboards, Projects, Issues, and Merge Requests ([introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54947) in GitLab 11.8). - The user uses the API - The user uses the GraphQL API diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 102eb1f8f53..0d028cfe77a 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Markdown This Markdown guide is **valid only for GitLab's internal Markdown rendering system for entries and files**. @@ -5,7 +11,7 @@ It is **not** valid for the [GitLab documentation website](https://docs.gitlab.c or [GitLab's main website](https://about.gitlab.com), as they both use [Kramdown](https://kramdown.gettalong.org) as their Markdown engine. The documentation website uses an extended Kramdown gem, [GitLab Kramdown](https://gitlab.com/gitlab-org/gitlab_kramdown). -Consult the [GitLab Kramdown Guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/) +Consult the [GitLab Kramdown Guide](https://about.gitlab.com/handbook/markdown-guide/) for a complete Kramdown reference. NOTE: **Note:** We encourage you to view this document as [rendered by GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md). @@ -76,7 +82,7 @@ NOTE: **Note:** We will flag any significant differences between Redcarpet and C If you have a large volume of Markdown files, it can be tedious to determine if they will display correctly or not. You can use the [diff_redcarpet_cmark](https://gitlab.com/digitalmoksha/diff_redcarpet_cmark) -tool (not an officially supported product) to generate a list of files, and the +tool (not an officially supported product) to generate a list of files and the differences between how RedCarpet and CommonMark render the files. It can give an indication if anything needs to be changed - often nothing will need to change. @@ -253,7 +259,7 @@ Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) > **Note:** The emoji example above uses hard-coded images for this documentation. The emoji, when rendered within GitLab, may appear different depending on the OS and browser used. -Most emoji are natively supported on macOS, Windows, iOS, Android and will fallback to image-based emoji where there is lack of support. +Most emoji are natively supported on macOS, Windows, iOS, Android, and will fall back on image-based emoji where there is no support. NOTE: **Note:** On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/) to get full native emoji support. Ubuntu 18.04 (like many modern Linux distributions) has @@ -272,7 +278,7 @@ in a box at the top of the document, before the rendered HTML content. To view a you can toggle between the source and rendered version of a [GitLab documentation file](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/README.md). In GitLab, front matter is only used in Markdown files and wiki pages, not the other -places where Markdown formatting is supported. It must be at the very top of the document, +places where Markdown formatting is supported. It must be at the very top of the document and must be between delimiters, as explained below. The following delimiters are supported: @@ -405,7 +411,7 @@ GFM recognizes special GitLab related references. For example, you can easily re an issue, a commit, a team member, or even the whole team within a project. GFM will turn that reference into a link so you can navigate between them easily. -Additionally, GFM recognizes certain cross-project references, and also has a shorthand +Additionally, GFM recognizes certain cross-project references and also has a shorthand version to reference other projects from the same namespace. GFM will recognize the following: @@ -435,8 +441,8 @@ GFM will recognize the following: In addition to this, links to some objects are also recognized and formatted. Some examples of these are: - Comments on issues: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234#note_101075757"`, which will be rendered as `#1234 (note1)` -- The issues designs tab: `"https://gitlab.com/gitlab-org/gitlab/issues/1234/designs"`, which will be rendered as `#1234 (designs)`. - **(PREMIUM)** +- The issues designs tab: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs"`, which will be rendered as `#1234 (designs)`. +- Links to individual designs: `"https://gitlab.com/gitlab-org/gitlab/-/issues/1234/designs/layout.png"`, which will be rendered as `#1234[layout.png]`. ### Task lists @@ -825,7 +831,7 @@ Regardless of the tag names, the relative order of the reference tags determines numbering. Reference tags can use letters and other characters. Avoid using lowercase `w` or an underscore -(`_`) in footnote tag names until [this bug](https://gitlab.com/gitlab-org/gitlab/issues/24423) is +(`_`) in footnote tag names until [this bug](https://gitlab.com/gitlab-org/gitlab/-/issues/24423) is resolved. <!-- @@ -1011,7 +1017,7 @@ You can also use raw HTML in your Markdown, and it will usually work pretty well See the documentation for HTML::Pipeline's [SanitizationFilter](https://github.com/jch/html-pipeline/blob/v2.12.3/lib/html/pipeline/sanitization_filter.rb#L42) class for the list of allowed HTML tags and attributes. In addition to the default -`SanitizationFilter` whitelist, GitLab allows `span`, `abbr`, `details` and `summary` elements. +`SanitizationFilter` allowlist, GitLab allows `span`, `abbr`, `details` and `summary` elements. ```html <dl> @@ -1103,6 +1109,11 @@ These details <em>will</em> remain <strong>hidden</strong> until expanded. Markdown inside these tags is supported as well. +NOTE: **Note:** +If your Markdown isn't rendering correctly, try adding +`{::options parse_block_html="true" /}` to the top of the page, and add +`markdown="span"` to the opening summary tag like this: `<summary markdown="span">`. + Remember to leave a blank line after the `</summary>` tag and before the `</details>` tag, as shown in the example: @@ -1427,7 +1438,9 @@ Example: Additionally, you can choose the alignment of text within columns by adding colons (`:`) to the sides of the "dash" lines in the second row. This will affect every cell in the column. -> Note that the headers are always right aligned [within GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#tables). +NOTE: **Note:** +[Within GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#tables), +the headers are always left-aligned in Chrome and Firefox, and centered in Safari. ```markdown | Left Aligned | Centered | Right Aligned | Left Aligned | Centered | Right Aligned | @@ -1443,7 +1456,7 @@ to the sides of the "dash" lines in the second row. This will affect every cell #### Copy from spreadsheet and paste in Markdown -[Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27205) in GitLab 12.7. +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27205) in GitLab 12.7. If you're working in spreadsheet software (for example, Microsoft Excel, Google Sheets, or Apple Numbers), you can copy from a spreadsheet, and GitLab will diff --git a/doc/user/operations_dashboard/index.md b/doc/user/operations_dashboard/index.md index 531422ca077..5a2ff410253 100644 --- a/doc/user/operations_dashboard/index.md +++ b/doc/user/operations_dashboard/index.md @@ -1,6 +1,6 @@ # Operations Dashboard **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5781) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.5. [Moved](https://gitlab.com/gitlab-org/gitlab/issues/9218) to [GitLab Premium](https://about.gitlab.com/pricing/) in 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5781) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.5. [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/9218) to [GitLab Premium](https://about.gitlab.com/pricing/) in 11.10. The Operations Dashboard provides a summary of each project's operational health, including pipeline and alert status. diff --git a/doc/user/packages/composer_repository/index.md b/doc/user/packages/composer_repository/index.md new file mode 100644 index 00000000000..8a7c70ec74d --- /dev/null +++ b/doc/user/packages/composer_repository/index.md @@ -0,0 +1,155 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# GitLab Composer Repository **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15886) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.1. + +With the GitLab Composer Repository, every project can have its own space to store [Composer](https://getcomposer.org/) packages. + +## Enabling the Composer Repository + +NOTE: **Note:** +This option is available only if your GitLab administrator has +[enabled support for the Package Registry](../../../administration/packages/index.md). **(PREMIUM ONLY)** + +After the Composer Repository is enabled, it will be available for all new projects +by default. To enable it for existing projects, or if you want to disable it: + +1. Navigate to your project's **Settings > General > Permissions**. +1. Find the Packages feature and enable or disable it. +1. Click on **Save changes** for the changes to take effect. + +You should then be able to see the **Packages & Registries** section on the left sidebar. + +## Getting started + +This section will cover creating a new example Composer package to publish. This is a +quickstart to test out the **GitLab Composer Registry**. + +You will need a recent version of [Composer](https://getcomposer.org/). + +### Creating a package project + +Understanding how to create a full Composer project is outside the scope of this +guide, but you can create a small package to test out the registry. Start by +creating a new directory called `my-composer-package`: + +```shell +mkdir my-composer-package && cd my-composer-package +``` + +Create a new `composer.json` file inside this directory to set up the basic project: + +```shell +touch composer.json +``` + +Inside `composer.json`, add the following code: + +```json +{ + "name": "<namespace>/composer-test", + "type": "library", + "license": "GPL-3.0-only", + "version": "1.0.0" +} +``` + +Replace `<namespace>` with a unique namespace like your GitLab username or group name. + +After this basic package structure is created, we need to tag it in Git and push it to the repository. + +```shell +git init +git add composer.json +git commit -m 'Composer package test' +git tag v1.0.0 +git add origin git@gitlab.com:<namespace>/<project-name>.git +git push origin v1.0.0 +``` + +### Publishing the package + +Now that the basics of our project is completed, we can publish the package. +For accomplishing this you will need the following: + +- A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. +- Your project ID which can be found on the home page of your project. + +To publish the package hosted on GitLab we'll need to make a `POST` to the GitLab package API using a tool like `curl`: + +```shell +curl --data tag=<tag> 'https://__token__:<personal-access-token>@gitlab.com/api/v4/projects/<project_id>/packages/composer' +``` + +Where: + +- `<personal-access-token>` is your personal access token. +- `<project_id>` is your project ID. +- `<tag>` is the Git tag name of the version you want to publish. In this example it should be `v1.0.0`. Notice that instead of `tag=<tag>` you can also use `branch=<branch>` to publish branches. + +If the above command succeeds, you now should be able to see the package under the **Packages & Registries** section of your project page. + +### Installing a package + +To install your package you will need: + +- A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication. +- Your group ID which can be found on the home page of your project's group. + +Add the GitLab Composer package repository to your existing project's `composer.json` file, along with the package name and version you want to install like so: + +```json +{ + ... + "repositories": [ + { "type": "composer", "url": "https://gitlab.com/api/v4/group/<group_id>/-/packages/composer/packages.json" } + ], + "require": { + ... + "<package_name>": "<version>" + }, + ... +} +``` + +Where: + +- `<group_id>` is the group ID found under your project's group page. +- `<package_name>` is your package name as defined in your package's `composer.json` file. +- `<version>` is your package version (`1.0.0` in this example). + +You will also need to create a `auth.json` file with your GitLab credentials: + +```json +{ + "http-basic": { + "gitlab.com": { + "username": "___token___", + "password": "<personal_access_token>" + } + } +} +``` + +Where: + +- `<personal_access_token>` is your personal access token. + +With the `composer.json` and `auth.json` files configured, you can install the package by running `composer`: + +```shell +composer update +``` + +If successful, you should be able to see the output indicating that the package has been successfully installed. + +CAUTION: **Important:** +Make sure to never commit the `auth.json` file to your repository. To install packages from a CI job, +consider using the [`composer config`](https://getcomposer.org/doc/articles/handling-private-packages-with-satis.md#authentication) tool with your personal access token +stored in a [GitLab CI/CD environment variable](../../../ci/variables/README.md) or in +[Hashicorp Vault](../../../ci/examples/authenticating-with-hashicorp-vault/index.md). diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index a56a67d4959..020028dfc10 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -1,6 +1,12 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Conan Repository **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8248) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8248) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. With the GitLab Conan Repository, every project can have its own space to store Conan packages. @@ -184,7 +190,7 @@ it is not possible due to a naming collision. For example: | `gitlab-org/gitlab-ce` | `my-package/1.0.0@foo/stable` | No | NOTE: **Note:** -A future iteration will extend support to [project and group level](https://gitlab.com/gitlab-org/gitlab/issues/11679) remotes which will allow for more flexible naming conventions. +A future iteration will extend support to [project and group level](https://gitlab.com/gitlab-org/gitlab/-/issues/11679) remotes which will allow for more flexible naming conventions. ## Installing a package @@ -271,7 +277,7 @@ The GitLab Conan repository supports the following Conan CLI commands: ## Using GitLab CI with Conan packages -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11678) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11678) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. To work with Conan commands within [GitLab CI/CD](./../../../ci/README.md), you can use `CI_JOB_TOKEN` in place of the personal access token in your commands. diff --git a/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_0.png b/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_0.png Binary files differdeleted file mode 100644 index 14119abd56a..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_0.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_1.png b/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_1.png Binary files differnew file mode 100644 index 00000000000..bbbba44eb9b --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_group_repositories_v13_1.png diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_v13_0.png b/doc/user/packages/container_registry/img/container_registry_repositories_v13_0.png Binary files differdeleted file mode 100644 index 7286007b953..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repositories_v13_0.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_v13_1.png b/doc/user/packages/container_registry/img/container_registry_repositories_v13_1.png Binary files differnew file mode 100644 index 00000000000..13a6d1a4470 --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_repositories_v13_1.png diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_0.png b/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_0.png Binary files differdeleted file mode 100644 index f7c3aafcc8e..00000000000 --- a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_0.png +++ /dev/null diff --git a/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_1.png b/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_1.png Binary files differnew file mode 100644 index 00000000000..35a02182a77 --- /dev/null +++ b/doc/user/packages/container_registry/img/container_registry_repositories_with_quickstart_v13_1.png diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index 5e642e1e21c..eb1933de62a 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Container Registry > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4040) in GitLab 8.8. @@ -20,7 +26,7 @@ have its own space to store its Docker images. You can read more about Docker Registry at <https://docs.docker.com/registry/introduction/>. - + ## Enable the Container Registry for your project @@ -56,7 +62,7 @@ for both projects and groups. Navigate to your project's **{package}** **Packages & Registries > Container Registry**. - + This view will: @@ -71,7 +77,7 @@ This view will: Navigate to your groups's **{package}** **Packages & Registries > Container Registry**. - + This view will: @@ -237,29 +243,29 @@ For private and internal projects: ### Container Registry examples with GitLab CI/CD -If you're using docker-in-docker on your Runners, this is how your `.gitlab-ci.yml` +If you're using Docker-in-Docker on your Runners, this is how your `.gitlab-ci.yml` should look similar to this: ```yaml build: - image: docker:19.03.8 + image: docker:19.03.11 stage: build services: - - docker:19.03.8-dind + - docker:19.03.11-dind script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker build -t $CI_REGISTRY/group/project/image:latest . - docker push $CI_REGISTRY/group/project/image:latest ``` -You can also make use of [other variables](../../../ci/variables/README.md) to avoid hardcoding: +You can also make use of [other variables](../../../ci/variables/README.md) to avoid hard-coding: ```yaml build: - image: docker:19.03.8 + image: docker:19.03.11 stage: build services: - - docker:19.03.8-dind + - docker:19.03.11-dind variables: IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG script: @@ -282,9 +288,9 @@ when needed. Changes to `master` also get tagged as `latest` and deployed using an application-specific deploy script: ```yaml -image: docker:19.03.8 +image: docker:19.03.11 services: - - docker:19.03.8-dind + - docker:19.03.11-dind stages: - build @@ -344,11 +350,11 @@ or [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html) execut make sure that [`pull_policy`](https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work) is set to `always`. -### Using a docker-in-docker image from your Container Registry +### Using a Docker-in-Docker image from your Container Registry -If you want to use your own Docker images for docker-in-docker, there are a few +If you want to use your own Docker images for Docker-in-Docker, there are a few things you need to do in addition to the steps in the -[docker-in-docker](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor) section: +[Docker-in-Docker](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor) section: 1. Update the `image` and `service` to point to your registry. 1. Add a service [alias](../../../ci/yaml/README.md#servicesalias). @@ -357,9 +363,9 @@ Below is an example of what your `.gitlab-ci.yml` should look like: ```yaml build: - image: $CI_REGISTRY/group/project/docker:19.03.8 + image: $CI_REGISTRY/group/project/docker:19.03.11 services: - - name: $CI_REGISTRY/group/project/docker:19.03.8-dind + - name: $CI_REGISTRY/group/project/docker:19.03.11-dind alias: docker stage: build script: @@ -367,7 +373,7 @@ Below is an example of what your `.gitlab-ci.yml` should look like: - docker run my-docker-image /script/to/run/tests ``` -If you forget to set the service alias, the `docker:19.03.8` image won't find the +If you forget to set the service alias, the `docker:19.03.11` image won't find the `dind` service, and an error like the following will be thrown: ```plaintext @@ -437,10 +443,10 @@ stages: - clean build_image: - image: docker:19.03.8 + image: docker:19.03.11 stage: build services: - - docker:19.03.8-dind + - docker:19.03.11-dind variables: IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG script: @@ -453,10 +459,10 @@ build_image: - master delete_image: - image: docker:19.03.8 + image: docker:19.03.11 stage: clean services: - - docker:19.03.8-dind + - docker:19.03.11-dind variables: IMAGE_TAG: $CI_PROJECT_PATH:$CI_COMMIT_REF_SLUG REG_SHA256: ade837fc5224acd8c34732bf54a94f579b47851cc6a7fd5899a98386b782e228 @@ -487,11 +493,12 @@ older tags and images are regularly removed from the Container Registry. ## Expiration policy -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/15398) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15398) in GitLab 12.8. NOTE: **Note:** -Expiration policies for projects created before GitLab 12.8 may be enabled by an -admin in the [CI/CD Package Registry settings](./../../admin_area/settings/index.md#cicd). +For GitLab.com, expiration policies are not available for projects created before GitLab 12.8. +For self-managed instances, expiration policies may be enabled by an admin in the +[CI/CD Package Registry settings](./../../admin_area/settings/index.md#cicd). Note the inherent [risks involved](./index.md#use-with-external-container-registries). It is possible to create a per-project expiration policy, so that you can make sure that @@ -506,7 +513,7 @@ then goes through a process of excluding tags from it until only the ones to be 1. Excludes any tags that do not have a manifest (not part of the options). 1. Orders the remaining tags by `created_date`. 1. Excludes from the list the N tags based on the `keep_n` value (Number of tags to retain). -1. Excludes from the list the tags older than the `older_than` value (Expiration interval). +1. Excludes from the list the tags more recent than the `older_than` value (Expiration interval). 1. Excludes from the list any tags matching the `name_regex_keep` value (Images to preserve). 1. Finally, the remaining tags in the list are deleted from the Container Registry. @@ -525,6 +532,17 @@ The UI allows you to configure the following: - **Docker tags with names matching this regex pattern will expire:** the regex used to determine what tags should be expired. To qualify all tags for expiration, use the default value of `.*`. - **Docker tags with names matching this regex pattern will be preserved:** the regex used to determine what tags should be preserved. To preserve all tags, use the default value of `.*`. +#### Troubleshooting expiration policies + +If you see the following message: + +"Something went wrong while updating the expiration policy." + +Check the regex patterns to ensure they are valid. + +You can use [Rubular](https://rubular.com/) to check your regex. +View some common [regex pattern examples](#regex-pattern-examples). + ### Managing project expiration policy through the API You can set, update, and disable the expiration policies using the GitLab API. @@ -548,6 +566,36 @@ run the policy may get backed up or fail completely. It is recommended you only policies for projects that were created before GitLab 12.8 if you are confident the amount of tags being cleaned up will be minimal. +### Regex pattern examples + +Expiration policies use regex patterns to determine which tags should be preserved or removed, both in the UI and the API. + +Here are examples of regex patterns you may want to use: + +- Match all tags: + + ```plaintext + .* + ``` + +- Match tags that start with `v`: + + ```plaintext + v.+ + ``` + +- Match tags that contain `master`: + + ```plaintext + master + ``` + +- Match tags that either start with `v`, contain `master`, or contain `release`: + + ```plaintext + (?:v.+|master|release) + ``` + ## Limitations - Moving or renaming existing Container Registry repositories is not supported diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index be9710053dd..e68883a1382 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -1,6 +1,12 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Dependency Proxy **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7934) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. NOTE: **Note:** This is the user guide. In order to use the dependency proxy, an administrator diff --git a/doc/user/packages/go_proxy/index.md b/doc/user/packages/go_proxy/index.md new file mode 100644 index 00000000000..a705b956d30 --- /dev/null +++ b/doc/user/packages/go_proxy/index.md @@ -0,0 +1,169 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# GitLab Go Proxy **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27376) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.1. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-the-go-proxy). **(PREMIUM)** + +With the Go proxy for GitLab, every project in GitLab can be fetched with the +[Go proxy protocol](https://proxy.golang.org/). + +## Prerequisites + +### Enable the Go proxy + +The Go proxy for GitLab is under development and not ready for production use, due to +[potential performance issues with large repositories](https://gitlab.com/gitlab-org/gitlab/-/issues/218083). + +It is deployed behind a feature flag that is **disabled by default**. + +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:go_proxy) # or +``` + +To disable it: + +```ruby +Feature.disable(:go_proxy) +``` + +To enable or disable it for specific projects: + +```ruby +Feature.enable(:go_proxy, Project.find(1)) +Feature.disable(:go_proxy, Project.find(2)) +``` + +### Enable the Package Registry + +The Package Registry is enabled for new projects by default. If you cannot find +the **{package}** **Packages > List** entry under your project's sidebar, verify +the following: + +1. Your GitLab administrator has + [enabled support for the Package Registry](../../../administration/packages/index.md). **(PREMIUM ONLY)** +1. The Package Registry is [enabled for your project](../index.md). + +NOTE: **Note:** +GitLab does not currently display Go modules in the **Packages Registry** of a project. +Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213770) for details. + +## Add GitLab as a Go proxy + +NOTE: **Note:** +To use a Go proxy, you must be using Go 1.13 or later. + +The available proxy endpoints are: + +- Project - can fetch modules defined by a project - `/api/v4/projects/:id/packages/go` + +To use the Go proxy for GitLab to fetch Go modules from GitLab, add the +appropriate proxy endpoint to `GOPROXY`. For details on setting Go environment +variables, see [Set environment variables](#set-environment-variables). For +details on configuring `GOPROXY`, see [Dependency Management in Go > +Proxies](../../../development/go_guide/dependencies.md#proxies). + +For example, adding the project-specific endpoint to `GOPROXY` will tell Go +to initially query that endpoint and fall back to the default behavior: + +```shell +go env -w GOPROXY='https://gitlab.com/api/v4/projects/1234/packages/go,https://proxy.golang.org,direct' +``` + +With this configuration, Go fetches dependencies as follows: + +1. Attempt to fetch from the project-specific Go proxy. +1. Attempt to fetch from [proxy.golang.org](https://proxy.golang.org). +1. Fetch directly with version control system operations (such as `git clone`, + `svn checkout`, and so on). + +If `GOPROXY` is not specified, Go follows steps 2 and 3, which corresponds to +setting `GOPROXY` to `https://proxy.golang.org,direct`. If `GOPROXY` only +contains the project-specific endpoint, Go will only query that endpoint. + +## Fetch modules from private projects + +`go` does not support transmitting credentials over insecure connections. The +steps below work only if GitLab is configured for HTTPS. + +1. Configure Go to include HTTP basic authentication credentials when fetching + from the Go proxy for GitLab. +1. Configure Go to skip downloading of checksums for private GitLab projects + from the public checksum database. + +### Enable request authentication + +Create a [personal access token](../../profile/personal_access_tokens.md) with +the `api` or `read_api` scope and add it to +[`~/.netrc`](https://ec.haxx.se/usingcurl/usingcurl-netrc): + +```netrc +machine <url> login <username> password <token> +``` + +`<url>` should be the URL of the GitLab instance, for example `gitlab.com`. +`<username>` and `<token>` should be your username and the personal access +token, respectively. + +### Disable checksum database queries + +When downloading dependencies, by default Go 1.13 and later validate fetched +sources against the checksum database `sum.golang.org`. If the checksum of the +fetched sources does not match the checksum from the database, Go will not build +the dependency. This causes private modules to fail to build, as +`sum.golang.org` cannot fetch the source of private modules and thus cannot +provide a checksum. To resolve this issue, `GONOSUMDB` should be set to a +comma-separated list of private projects. For details on setting Go environment +variables, see [Set environment variables](#set-environment-variables). For more +details on disabling this feature of Go, see [Dependency Management in Go > +Checksums](../../../development/go_guide/dependencies.md#checksums). + +For example, to disable checksum queries for `gitlab.com/my/project`, set `GONOSUMDB`: + +```shell +go env -w GONOSUMDB='gitlab.com/my/project,<previous value>' +``` + +## Working with Go + +If you are unfamiliar with managing dependencies in Go, or Go in general, +consider reviewing the following documentation: + +- [Dependency Management in Go](../../../development/go_guide/dependencies.md) +- [Go Modules Reference](https://golang.org/ref/mod) +- [Documentation (golang.org)](https://golang.org/doc/) +- [Learn (learn.go.dev)](https://learn.go.dev/) + +### Set environment variables + +Go uses environment variables to control various features. These can be managed +in all the usual ways, but Go 1.14 will read and write Go environment variables +from and to a special Go environment file, `~/.go/env` by default. If `GOENV` is +set to a file, Go will read and write that file instead. If `GOENV` is not set +but `GOPATH` is set, Go will read and write `$GOPATH/env`. + +Go environment variables can be read with `go env <var>` and, in Go 1.14 and +later, can be written with `go env -w <var>=<value>`. For example, `go env +GOPATH` or `go env -w GOPATH=/go`. + +### Release a module + +Go modules and module versions are defined by source repositories, such as Git, +SVN, Mercurial, and so on. A module is a repository containing `go.mod` and Go +files. Module versions are defined by VCS tags. To publish a module, push +`go.mod` and source files to a VCS repository. To publish a module version, push +a VCS tag. See [Dependency Management in Go > +Versioning](../../../development/go_guide/dependencies.md#versioning) for more +details on what constitutes a valid module or module version. diff --git a/doc/user/packages/index.md b/doc/user/packages/index.md index 132a64d99a3..6e59a87ae36 100644 --- a/doc/user/packages/index.md +++ b/doc/user/packages/index.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Package Registry GitLab Packages allows organizations to utilize GitLab as a private repository @@ -15,6 +21,8 @@ The Packages feature allows GitLab to act as a repository for the following: | [NPM Registry](npm_registry/index.md) **(PREMIUM)** | The GitLab NPM Registry enables every project in GitLab to have its own space to store [NPM](https://www.npmjs.com/) packages. | 11.7+ | | [NuGet Repository](nuget_repository/index.md) **(PREMIUM)** | The GitLab NuGet Repository will enable every project in GitLab to have its own space to store [NuGet](https://www.nuget.org/) packages. | 12.8+ | | [PyPi Repository](pypi_repository/index.md) **(PREMIUM)** | The GitLab PyPi Repository will enable every project in GitLab to have its own space to store [PyPi](https://pypi.org/) packages. | 12.10+ | +| [Go Proxy](go_proxy/index.md) **(PREMIUM)** | The Go proxy for GitLab enables every project in GitLab to be fetched with the [Go proxy protocol](https://proxy.golang.org/). | 13.1+ | +| [Composer Repository](composer_repository/index.md) **(PREMIUM)** | The GitLab Composer Repository will enable every project in GitLab to have its own space to store [Composer](https://getcomposer.org/) packages. | 13.1+ | ## Enable the Package Registry for your project @@ -104,20 +112,19 @@ are adding support for [PHP](https://gitlab.com/gitlab-org/gitlab/-/merge_reques | Format | Use case | | ------ | ------ | -| [Cargo](https://gitlab.com/gitlab-org/gitlab/issues/33060) | Cargo is the Rust package manager. Build, publish and share Rust packages | -| [Chef](https://gitlab.com/gitlab-org/gitlab/issues/36889) | Configuration management with Chef using all the benefits of a repository manager. | -| [CocoaPods](https://gitlab.com/gitlab-org/gitlab/issues/36890) | Speed up development with Xcode and CocoaPods. | -| [Conda](https://gitlab.com/gitlab-org/gitlab/issues/36891) | Secure and private local Conda repositories. | -| [CRAN](https://gitlab.com/gitlab-org/gitlab/issues/36892) | Deploy and resolve CRAN packages for the R language. | -| [Debian](https://gitlab.com/gitlab-org/gitlab/issues/5835) | Host and provision Debian packages. | -| [Go](https://gitlab.com/gitlab-org/gitlab/issues/9773) | Resolve Go dependencies from and publish your Go packages to GitLab. | -| [Opkg](https://gitlab.com/gitlab-org/gitlab/issues/36894) | Optimize your work with OpenWrt using Opkg repositories. | -| [P2](https://gitlab.com/gitlab-org/gitlab/issues/36895) | Host all your Eclipse plugins in your own GitLab P2 repository. | -| [Puppet](https://gitlab.com/gitlab-org/gitlab/issues/36897) | Configuration management meets repository management with Puppet repositories. | -| [RPM](https://gitlab.com/gitlab-org/gitlab/issues/5932) | Distribute RPMs directly from GitLab. | -| [RubyGems](https://gitlab.com/gitlab-org/gitlab/issues/803) | Use GitLab to host your own gems. | -| [SBT](https://gitlab.com/gitlab-org/gitlab/issues/36898) | Resolve dependencies from and deploy build output to SBT repositories when running SBT builds. | -| [Vagrant](https://gitlab.com/gitlab-org/gitlab/issues/36899) | Securely host your Vagrant boxes in local repositories. | +| [Cargo](https://gitlab.com/gitlab-org/gitlab/-/issues/33060) | Cargo is the Rust package manager. Build, publish and share Rust packages | +| [Chef](https://gitlab.com/gitlab-org/gitlab/-/issues/36889) | Configuration management with Chef using all the benefits of a repository manager. | +| [CocoaPods](https://gitlab.com/gitlab-org/gitlab/-/issues/36890) | Speed up development with Xcode and CocoaPods. | +| [Conda](https://gitlab.com/gitlab-org/gitlab/-/issues/36891) | Secure and private local Conda repositories. | +| [CRAN](https://gitlab.com/gitlab-org/gitlab/-/issues/36892) | Deploy and resolve CRAN packages for the R language. | +| [Debian](https://gitlab.com/gitlab-org/gitlab/-/issues/5835) | Host and provision Debian packages. | +| [Opkg](https://gitlab.com/gitlab-org/gitlab/-/issues/36894) | Optimize your work with OpenWrt using Opkg repositories. | +| [P2](https://gitlab.com/gitlab-org/gitlab/-/issues/36895) | Host all your Eclipse plugins in your own GitLab P2 repository. | +| [Puppet](https://gitlab.com/gitlab-org/gitlab/-/issues/36897) | Configuration management meets repository management with Puppet repositories. | +| [RPM](https://gitlab.com/gitlab-org/gitlab/-/issues/5932) | Distribute RPMs directly from GitLab. | +| [RubyGems](https://gitlab.com/gitlab-org/gitlab/-/issues/803) | Use GitLab to host your own gems. | +| [SBT](https://gitlab.com/gitlab-org/gitlab/-/issues/36898) | Resolve dependencies from and deploy build output to SBT repositories when running SBT builds. | +| [Vagrant](https://gitlab.com/gitlab-org/gitlab/-/issues/36899) | Securely host your Vagrant boxes in local repositories. | ## Package workflows diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 51e62dc871e..2d40a623fb8 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -1,6 +1,12 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab Maven Repository **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5811) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5811) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.3. With the GitLab [Maven](https://maven.apache.org) Repository, every project can have its own space to store its Maven artifacts. diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index b909646431b..4d60c227ccd 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -1,6 +1,12 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab NPM Registry **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.7. With the GitLab NPM Registry, every project can have its own space to store NPM packages. @@ -159,7 +165,7 @@ Then, you could run `npm publish` either locally or via GitLab CI/CD: ### Authenticating with a CI job token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9104) in GitLab Premium 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9104) in GitLab Premium 12.5. If you’re using NPM with GitLab CI/CD, a CI job token can be used instead of a personal access token or deploy token. The token will inherit the permissions of the user that generates the pipeline. @@ -272,7 +278,7 @@ yarn add @my-project-scope/my-package ### Forwarding requests to npmjs.org -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/55344) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/55344) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9. By default, when an NPM package is not found in the GitLab NPM Registry, the request will be forwarded to [npmjs.com](https://www.npmjs.com/). @@ -374,7 +380,7 @@ NPM_TOKEN=<your_token> npm install ## NPM dependencies metadata -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11867) in GitLab Premium 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11867) in GitLab Premium 12.6. Starting from GitLab 12.6, new packages published to the GitLab NPM Registry expose the following attributes to the NPM client: @@ -390,7 +396,7 @@ Starting from GitLab 12.6, new packages published to the GitLab NPM Registry exp ## NPM distribution tags -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9425) in GitLab Premium 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9425) in GitLab Premium 12.8. You can add [distribution tags](https://docs.npmjs.com/cli/dist-tag) for newly published packages. They follow NPM's convention where they are optional, and each tag can only be assigned to one diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index d9efb3239a8..6ada68dcceb 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -1,6 +1,12 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab NuGet Repository **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/20050) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20050) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. With the GitLab NuGet Repository, every project can have its own space to store NuGet packages. @@ -95,7 +101,7 @@ Where: For example: ```shell -nuget source Add -Name "GitLab" -Source "https//gitlab.example/api/v4/projects/10/packages/nuget/index.json" -UserName carol -Password 12345678asdf +nuget source Add -Name "GitLab" -Source "https://gitlab.example/api/v4/projects/10/packages/nuget/index.json" -UserName carol -Password 12345678asdf ``` ### Add NuGet Repository source with Visual Studio diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index 979f7a3c966..6f6609d82b1 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -1,3 +1,9 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # GitLab PyPi Repository **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208747) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. diff --git a/doc/user/permissions.md b/doc/user/permissions.md index e5893b291dc..ecae119e9f1 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -23,9 +23,6 @@ GitLab [administrators](../administration/index.md) receive all permissions. To add or import a user, you can follow the [project members documentation](project/members/index.md). -For information on eligible approvers for Merge Requests, see -[Eligible approvers](project/merge_requests/merge_request_approvals.md#eligible-approvers). - ## Principles behind permissions See our [product handbook on permissions](https://about.gitlab.com/handbook/product/#permissions-in-gitlab) @@ -50,7 +47,7 @@ The following table depicts the various user permission levels in a project. |---------------------------------------------------|---------|------------|-------------|----------|--------| | Download project | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | Leave comments | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | -| View approved/blacklisted licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | +| View allowed and denied licenses **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View License Compliance reports **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | | View Security reports **(ULTIMATE)** | ✓ (*3*) | ✓ | ✓ | ✓ | ✓ | | View Dependency list **(ULTIMATE)** | ✓ (*1*) | ✓ | ✓ | ✓ | ✓ | @@ -99,6 +96,7 @@ The following table depicts the various user permission levels in a project. | Assign merge requests | | | ✓ | ✓ | ✓ | | Label merge requests | | | ✓ | ✓ | ✓ | | Lock merge request threads | | | ✓ | ✓ | ✓ | +| Approve merge requests (*9*) | | | ✓ | ✓ | ✓ | | Manage/Accept merge requests | | | ✓ | ✓ | ✓ | | Create new environments | | | ✓ | ✓ | ✓ | | Stop environments | | | ✓ | ✓ | ✓ | @@ -131,6 +129,7 @@ The following table depicts the various user permission levels in a project. | Enable/disable tag protections | | | | ✓ | ✓ | | Edit project | | | | ✓ | ✓ | | Edit project badges | | | | ✓ | ✓ | +| Share (invite) projects with groups | | | | ✓ (*8*) | ✓ (*8*)| | Add deploy keys to project | | | | ✓ | ✓ | | Configure project hooks | | | | ✓ | ✓ | | Manage Runners | | | | ✓ | ✓ | @@ -150,9 +149,12 @@ The following table depicts the various user permission levels in a project. | Manage [project access tokens](./project/settings/project_access_tokens.md) **(CORE ONLY)** | | | | ✓ | ✓ | | Switch visibility level | | | | | ✓ | | Transfer project to another namespace | | | | | ✓ | +| Rename project | | | | | ✓ | | Remove fork relationship | | | | | ✓ | | Remove project | | | | | ✓ | +| Archive project | | | | | ✓ | | Delete issues | | | | | ✓ | +| Delete merge request | | | | | ✓ | | Disable notification emails | | | | | ✓ | | Force push to protected branches (*4*) | | | | | | | Remove protected branches (*4*) | | | | | | @@ -172,6 +174,9 @@ The following table depicts the various user permission levels in a project. 1. If the [branch is protected](./project/protected_branches.md#using-the-allowed-to-merge-and-allowed-to-push-settings), this depends on the access Developers and Maintainers are given. 1. Guest users can access GitLab [**Releases**](project/releases/index.md) for downloading assets but are not allowed to download the source code nor see repository information like tags and commits. 1. Actions are limited only to records owned (referenced) by user. +1. When [Share Group Lock](./group/index.md#share-with-group-lock) is enabled the project can't be shared with other groups. It does not affect group with group sharing. +1. For information on eligible approvers for merge requests, see + [Eligible approvers](project/merge_requests/merge_request_approvals.md#eligible-approvers). ## Project features permissions @@ -239,6 +244,7 @@ group. | Publish [packages](packages/index.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ | | View metrics dashboard annotations | | ✓ | ✓ | ✓ | ✓ | | Create project in group | | | ✓ (3) | ✓ (3) | ✓ (3) | +| Share (invite) groups with groups | | | | | ✓ | | Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | | Enable/disable a dependency proxy **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Use security dashboard **(ULTIMATE)** | | | ✓ | ✓ | ✓ | @@ -246,11 +252,12 @@ group. | View/manage group-level Kubernetes cluster | | | | ✓ | ✓ | | Create subgroup | | | | ✓ (1) | ✓ | | Edit epic comments (posted by any user) **(ULTIMATE)** | | | | ✓ (2) | ✓ (2) | -| Edit group | | | | | ✓ | +| Edit group settings | | | | | ✓ | | Manage group level CI/CD variables | | | | | ✓ | | Manage group members | | | | | ✓ | -| Remove group | | | | | ✓ | +| Delete group | | | | | ✓ | | Delete group epic **(ULTIMATE)** | | | | | ✓ | +| Edit SAML SSO Billing **(SILVER ONLY)** | ✓ | ✓ | ✓ | ✓ | ✓ (4) | | View group Audit Events | | | | | ✓ | | Disable notification emails | | | | | ✓ | | View Contribution analytics | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -258,6 +265,8 @@ group. | View Issues analytics **(PREMIUM)** | ✓ | ✓ | ✓ | ✓ | ✓ | | View Productivity analytics **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | | View Value Stream analytics | ✓ | ✓ | ✓ | ✓ | ✓ | +| View Billing **(FREE ONLY)** | ✓ | ✓ | ✓ | ✓ | ✓ (4) | +| View Usage Quotas **(FREE ONLY)** | ✓ | ✓ | ✓ | ✓ | ✓ (4) | 1. Groups can be set to [allow either Owners or Owners and Maintainers to create subgroups](group/subgroups/index.md#creating-a-subgroup) @@ -265,6 +274,7 @@ group. 1. Default project creation role can be changed at: - The [instance level](admin_area/settings/visibility_and_access_controls.md#default-project-creation-protection). - The [group level](group/index.md#default-project-creation-level). +1. Does not apply to subgroups. ### Subgroup permissions @@ -285,7 +295,7 @@ project and should only have access to that project. External users: - Cannot create groups, projects, or personal snippets. -- Can only access projects to which they are explicitly granted access, +- Can only access public projects and projects to which they are explicitly granted access, thus hiding all other internal or private ones from them (like being logged out). @@ -455,7 +465,7 @@ for details about the pipelines security model. ## LDAP users permissions Since GitLab 8.15, LDAP user permissions can now be manually overridden by an admin user. -Read through the documentation on [LDAP users permissions](../administration/auth/how_to_configure_ldap_gitlab_ee/index.md) to learn more. +Read through the documentation on [LDAP users permissions](group/index.md#manage-group-memberships-via-ldap) to learn more. ## Project aliases diff --git a/doc/user/profile/account/create_accounts.md b/doc/user/profile/account/create_accounts.md index 27aa57e7f99..26c2c1bed89 100644 --- a/doc/user/profile/account/create_accounts.md +++ b/doc/user/profile/account/create_accounts.md @@ -1,5 +1,8 @@ --- type: reference +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Creating users **(CORE ONLY)** @@ -32,5 +35,5 @@ You can also [create users through the API](../../../api/users.md) as an admin. Users will be: -- Automatically created upon first login with the [LDAP integration](../../../administration/auth/ldap.md). +- Automatically created upon first login with the [LDAP integration](../../../administration/auth/ldap/index.md). - Created when first logging in via an [OmniAuth provider](../../../integration/omniauth.md) if the `allow_single_sign_on` setting is present. diff --git a/doc/user/profile/account/delete_account.md b/doc/user/profile/account/delete_account.md index c9193c6d94c..3c6f2989091 100644 --- a/doc/user/profile/account/delete_account.md +++ b/doc/user/profile/account/delete_account.md @@ -1,5 +1,8 @@ --- type: howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Deleting a User account diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index ac0835911d2..bfcaeaf6a15 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -1,5 +1,8 @@ --- type: howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Two-Factor Authentication @@ -65,19 +68,22 @@ in a safe place. ### Enable 2FA via U2F device +> Introduced in [GitLab 8.9](https://about.gitlab.com/blog/2016/06/22/gitlab-adds-support-for-u2f/). + GitLab officially only supports [YubiKey](https://www.yubico.com/products/) -U2F devices, but users have successfully used [SoloKeys](https://solokeys.com/). +U2F devices, but users have successfully used [SoloKeys](https://solokeys.com/) +or [Google Titan Security Key](https://cloud.google.com/titan-security-key). The U2F workflow is [supported by](https://caniuse.com/#search=U2F) the following desktop browsers: - Chrome - Edge -- Firefox (disabled by default) +- Firefox 67+ - Opera NOTE: **Note:** -For Firefox, you can enable the FIDO U2F API in +For Firefox 47-66, you can enable the FIDO U2F API in [about:config](https://support.mozilla.org/en-US/kb/about-config-editor-firefox). Search for `security.webauth.u2f` and double click on it to toggle to `true`. diff --git a/doc/user/profile/active_sessions.md b/doc/user/profile/active_sessions.md index 408276127a2..4dbb11b581d 100644 --- a/doc/user/profile/active_sessions.md +++ b/doc/user/profile/active_sessions.md @@ -19,7 +19,7 @@ review the sessions, and revoke any you don't recognize. ## Active sessions limit -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31611) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31611) in GitLab 12.6. GitLab allows users to have up to 100 active sessions at once. If the number of active sessions exceeds 100, the oldest ones are deleted. diff --git a/doc/user/profile/img/unknown_sign_in_email_v13_1.png b/doc/user/profile/img/unknown_sign_in_email_v13_1.png Binary files differnew file mode 100644 index 00000000000..586be483be9 --- /dev/null +++ b/doc/user/profile/img/unknown_sign_in_email_v13_1.png diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index 383c7fe73aa..663a2888ee7 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -1,5 +1,8 @@ --- type: index, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # User account @@ -147,7 +150,7 @@ To add links to other accounts: ## Private contributions -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/14078) in GitLab 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14078) in GitLab 11.3. Enabling private contributions will include contributions to private projects, in the user contribution calendar graph and user recent activity. @@ -250,7 +253,27 @@ When the `_gitlab_session` expires or isn't available, GitLab uses the `remember to get you a new `_gitlab_session` and keep you signed in through browser restarts. After your `remember_user_token` expires and your `_gitlab_session` is cleared/expired, -you will be asked to sign in again to verify your identity (which is for security reasons). +you will be asked to sign in again to verify your identity for security reasons. + +### Increased sign-in time + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20340) in GitLab 13.1. + +The `remember_user_token` lifetime of a cookie can now extend beyond the deadline set by `config.remember_for`, as the `config.extend_remember_period` flag is now set to true. + +GitLab uses both session and persistent cookies: + +- Session cookie: Session cookies are normally removed at the end of the browser session when the browser is closed. The `_gitlab_session` cookie has no expiration date. +- Persistent cookie: The `remember_me_token` is a cookie with an expiration date of two weeks. GitLab activates this cookie if you click Remember Me when you sign in. + +By default, the server sets a time-to-live (TTL) of 1-week on any session that is used. + +When you close a browser, the session cookie may still remain. For example, Chrome has the "Continue where you left off" option that restores session cookies. +In other words, as long as you access GitLab at least once every 2 weeks, you could remain signed in with GitLab, as long as your browser tab is open. +The server continues to reset the TTL for that session, independent of whether 2FA is installed, +If you close your browser and open it up again, the `remember_user_token` cookie allows your user to reauthenticate itself. + +Without the `config.extend_remember_period` flag, you would be forced to sign in again after two weeks. <!-- ## Troubleshooting diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index ae00f3ace57..ee228050945 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -1,5 +1,8 @@ --- disqus_identifier: 'https://docs.gitlab.com/ee/workflow/notifications.html' +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab Notification Emails @@ -163,7 +166,7 @@ In most of the below cases, the notification will be sent to: - Custom: Users with notification level "custom" who turned on notifications for any of the events present in the table below NOTE: **Note:** -To minimize the number of notifications that do not require any action, from [GitLab 12.9 onwards](https://gitlab.com/gitlab-org/gitlab/issues/616), eligible approvers are no longer notified for all the activities in their projects. To receive them they have to change their user notification settings to **Watch** instead. +To minimize the number of notifications that do not require any action, from [GitLab 12.9 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/616), eligible approvers are no longer notified for all the activities in their projects. To receive them they have to change their user notification settings to **Watch** instead. | Event | Sent to | |------------------------|---------| @@ -237,4 +240,4 @@ reason `assigned` will have this sentence in the footer: - `You are receiving this email because you have been assigned an item on <configured GitLab hostname>.` NOTE: **Note:** -Notification of other events is being considered for inclusion in the `X-GitLab-NotificationReason` header. For details, see this [related issue](https://gitlab.com/gitlab-org/gitlab/issues/20689). +Notification of other events is being considered for inclusion in the `X-GitLab-NotificationReason` header. For details, see this [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/20689). diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 87c1fe4007a..e2c3dc74cf1 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -1,5 +1,8 @@ --- type: concepts, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Personal access tokens @@ -56,6 +59,58 @@ the following table. | `read_repository` | [GitLab 10.7](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17894) | Allows read-only access (pull) to the repository through `git clone`. | | `write_repository` | [GitLab 11.11](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26021) | Allows read-write access (pull, push) to the repository through `git clone`. Required for accessing Git repositories over HTTP when 2FA is enabled. | +## Programmatically creating a personal access token + +You can programmatically create a predetermined personal access token for use in +automation or tests. You will need sufficient access to run a +[Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) +for your GitLab instance. + +To create a token belonging to a user with username `automation-bot`, run the +following in the Rails console (`sudo gitlab-rails console`): + +```ruby +user = User.find_by_username('automation-bot') +token = user.personal_access_tokens.create(scopes: [:read_user, :read_repository], name: 'Automation token') +token.set_token('token-string-here123') +token.save! +``` + +This can be shortened into a single-line shell command using the +[GitLab Rails Runner](../../administration/troubleshooting/debug.md#using-the-rails-runner): + +```shell +sudo gitlab-rails runner "token = User.find_by_username('automation-bot').personal_access_tokens.create(scopes: [:read_user, :read_repository], name: 'Automation token'); token.set_token('token-string-here123'); token.save!" +``` + +NOTE: **Note:** +The token string must be 20 characters in length, or it will not be +recognized as a personal access token. + +The list of valid scopes and what they do can be found +[in the source code](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/auth.rb). + +## Programmatically revoking a personal access token + +You can programmatically revoke a personal access token. You will need +sufficient access to run a [Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session) +for your GitLab instance. + +To revoke a known token `token-string-here123`, run the following in the Rails +console (`sudo gitlab-rails console`): + +```ruby +token = PersonalAccessToken.find_by_token('token-string-here123') +token.revoke! +``` + +This can be shorted into a single-line shell command using the +[GitLab Rails Runner](../../administration/troubleshooting/debug.md#using-the-rails-runner): + +```shell +sudo gitlab-rails runner "PersonalAccessToken.find_by_token('token-string-here123').revoke!" +``` + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index 55781b48a27..ccaea61ae4b 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -36,6 +36,29 @@ The default theme is Indigo. You can choose between 10 themes:  +## Dark mode + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28252) in GitLab 13.1 as an Alpha release. + +GitLab has started work on dark mode! The dark mode Alpha release is available in the +spirit of iteration and the lower expectations of +[Alpha versions](https://about.gitlab.com/handbook/product/#alpha). + +Progress on dark mode is tracked in the [Dark theme epic](https://gitlab.com/groups/gitlab-org/-/epics/2902). See the epic for: + +- A list of known issues. +- Our planned direction and next steps. + +If you find an issue that isn’t listed, please leave a comment on the epic or create a +new issue. + +Dark mode is available as a navigation theme, for MVC and compatibility reasons. In +the future, we plan to make it configurable in its own section along with support for +[different navigation themes](https://gitlab.com/gitlab-org/gitlab/-/issues/219512). + +NOTE: **Note:** +Dark theme currently only works with the 'Dark' syntax highlighting. + ## Syntax highlighting theme NOTE: **Note:** diff --git a/doc/user/profile/unknown_sign_in_notification.md b/doc/user/profile/unknown_sign_in_notification.md index 9400ead1922..200358bb050 100644 --- a/doc/user/profile/unknown_sign_in_notification.md +++ b/doc/user/profile/unknown_sign_in_notification.md @@ -1,5 +1,14 @@ +--- +type: concepts, howto +stage: Manage +group: Access +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Email notification for unknown sign-ins +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27211) in GitLab 13.0. + When a user successfully signs in from a previously unknown IP address, GitLab notifies the user by email. In this way, GitLab proactively alerts users of potentially malicious or unauthorized sign-ins. @@ -13,4 +22,4 @@ There are two methods used to identify a known sign-in: ## Example email - + diff --git a/doc/user/project/badges.md b/doc/user/project/badges.md index 6994f16cf52..542a3763008 100644 --- a/doc/user/project/badges.md +++ b/doc/user/project/badges.md @@ -1,6 +1,6 @@ # Badges -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41174) in GitLab 10.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41174) in GitLab 10.7. Badges are a unified way to present condensed pieces of information about your projects. They consist of a small image and additionally a URL that the image diff --git a/doc/user/project/bulk_editing.md b/doc/user/project/bulk_editing.md index f4733620640..6d091c431a2 100644 --- a/doc/user/project/bulk_editing.md +++ b/doc/user/project/bulk_editing.md @@ -1,32 +1,60 @@ -# Bulk editing issues and merge requests +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- -> **Notes:** -> -> - A permission level of `Reporter` or higher is required in order to manage -> issues. -> - A permission level of `Developer` or higher is required in order to manage -> merge requests. +# Bulk editing issues and merge requests at the project level -Attributes can be updated simultaneously across multiple issues or merge requests -by using the bulk editing feature. +NOTE: **Note:** +Bulk editing issues, epics, and merge requests is also available at the **group level**. +For more details, see +[Bulk editing issues, epics, and merge requests at the group level](../group/bulk_editing/index.md). + +If you want to update attributes across multiple issues or merge requests, you can do it +by bulk editing them, that is, editing them together.  +## Bulk edit issues at the project level + NOTE: **Note:** -Bulk editing issues and merge requests is also available at the group level. -For more details, see [bulk editing group issues and merge requests](../group/bulk_editing/index.md). +You need a permission level of [Reporter or higher](../permissions.md) to manage issues. + +When bulk editing issues in a project, you can edit the following attributes: + +- Status (open/closed) +- Assignee +- Milestone +- Labels +- Subscriptions -To update multiple project issues or merge requests at the same time: +To update multiple project issues at the same time: -1. Navigate to their respective list. +1. In a project, go to **{issues}** **Issues > List**. +1. Click **Edit issues**. A sidebar on the right-hand side of your screen appears with editable fields. +1. Select the checkboxes next to each issue you want to edit. +1. Select the appropriate fields and their values from the sidebar. +1. Click **Update all**. + +## Bulk edit merge requests at the project level + +NOTE: **Note:** +You need a permission level of [Developer or higher](../permissions.md) to manage merge requests. -1. Click **Edit issues** or **Edit merge requests**. +When bulk editing merge requests in a project, you can edit the following attributes: - - This will open a sidebar on the right-hand side of your screen - where editable fields will be displayed. +- Status (open/closed) +- Assignee +- Milestone +- Labels +- Subscriptions - - Checkboxes will also appear beside each issue or merge request. +To update multiple project merge requests at the same time: -1. Check the checkboxes of each items to be edited. -1. Choose the appropriate fields and their values from the sidebar. +1. In a project, go to **{merge-request}** **Merge Requests**. +1. Click **Edit merge requests**. A sidebar on the right-hand side of your screen appears with + editable fields. +1. Select the checkboxes next to each merge request you want to edit. +1. Select the appropriate fields and their values from the sidebar. 1. Click **Update all**. diff --git a/doc/user/project/canary_deployments.md b/doc/user/project/canary_deployments.md index 86e9df1d162..852baf1f628 100644 --- a/doc/user/project/canary_deployments.md +++ b/doc/user/project/canary_deployments.md @@ -1,6 +1,6 @@ # Canary Deployments **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1659) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1659) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.1. A popular [Continuous Deployment](https://en.wikipedia.org/wiki/Continuous_deployment) strategy, where a small portion of the fleet is updated to the new version of diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index 712f8ea0adc..b11483a7446 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -51,7 +51,7 @@ Generate an access key for the IAM user, and configure GitLab with the credentia ## New EKS cluster -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/22392) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22392) in GitLab 12.5. To create and add a new Kubernetes cluster to your project, group, or instance: @@ -147,7 +147,7 @@ To create and add a new Kubernetes cluster to your project, group, or instance: - **VPC** - Select a [VPC](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html) to use for your EKS Cluster resources. - **Subnets** - Choose the [subnets](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html) - in your VPC where your worker nodes will run. + in your VPC where your worker nodes will run. You must select at least two. - **Security group** - Choose the [security group](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html) to apply to the EKS-managed Elastic Network Interfaces that are created in your worker node subnets. - **Instance type** - The [instance type](https://aws.amazon.com/ec2/instance-types/) of your worker nodes. @@ -164,114 +164,45 @@ You will need to add your AWS external ID to the [IAM Role in the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html#cli-configure-role-xaccount) to manage your cluster using `kubectl`. +### Troubleshooting creating a new cluster + +The following errors are commonly encountered when creating a new cluster. + +#### Error: Request failed with status code 422 + +When submitting the initial authentication form, GitLab returns a status code 422 +error when it can't determine the role you've provided. Make sure you've +correctly configured your role with the **Account ID** and **External ID** +provided by GitLab. In GitLab, make sure to enter the correct **Role ARN**. + +#### Could not load Security Groups for this VPC + +When populating options in the configuration form, GitLab returns this error +because GitLab has successfully assumed your provided role, but the role has +insufficient permissions to retrieve the resources needed for the form. Make sure +you've assigned the role the correct permissions. + +#### `ROLLBACK_FAILED` during cluster creation + +The creation process halted because GitLab encountered an error when creating +one or more resources. You can inspect the associated +[CloudFormation stack](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-view-stack-data-resources.html) +to find the specific resources that failed to create. + +If the `Cluster` resource failed with the error +`The provided role doesn't have the Amazon EKS Managed Policies associated with it.`, +the role specified in **Role name** is not configured correctly. + +NOTE: **Note:** +This role should not be the same as the one created above. If you don't have an +existing +[EKS cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html), +you must create one. + ## Existing EKS cluster -To add an existing EKS cluster to your project, group, or instance: - -1. Perform the following steps on the EKS cluster: - 1. Retrieve the certificate. A valid Kubernetes certificate is needed to authenticate to the - EKS cluster. We will use the certificate created by default. - Open a shell and use `kubectl` to retrieve it: - - 1. List the secrets with `kubectl get secrets`, and one should named similar to - `default-token-xxxxx`. Copy that token name for use below. - 1. Get the certificate with: - - ```shell - kubectl get secret <secret name> -o jsonpath="{['data']['ca\.crt']}" | base64 --decode - ``` - - 1. Create admin token. A `cluster-admin` token is required to install and manage Helm Tiller. - GitLab establishes mutual SSL authentication with Helm Tiller and creates limited service - accounts for each application. To create the token we will create an admin service account as - follows: - - 1. Create a file called `eks-admin-service-account.yaml` with contents: - - ```yaml - apiVersion: v1 - kind: ServiceAccount - metadata: - name: eks-admin - namespace: kube-system - ``` - - 1. Apply the service account to your cluster: - - ```shell - $ kubectl apply -f eks-admin-service-account.yaml - serviceaccount "eks-admin" created - ``` - - 1. Create a file called `eks-admin-cluster-role-binding.yaml` with contents: - - ```yaml - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRoleBinding - metadata: - name: eks-admin - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin - subjects: - - kind: ServiceAccount - name: eks-admin - namespace: kube-system - ``` - - 1. Apply the cluster role binding to your cluster: - - ```shell - $ kubectl apply -f eks-admin-cluster-role-binding.yaml - clusterrolebinding "eks-admin" created - ``` - - 1. Retrieve the token for the `eks-admin` service account: - - ```shell - kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep eks-admin | awk '{print $1}') - ``` - - Copy the `<authentication_token>` value from the output: - - ```yaml - Name: eks-admin-token-b5zv4 - Namespace: kube-system - Labels: <none> - Annotations: kubernetes.io/service-account.name=eks-admin - kubernetes.io/service-account.uid=bcfe66ac-39be-11e8-97e8-026dce96b6e8 - - Type: kubernetes.io/service-account-token - - Data - ==== - ca.crt: 1025 bytes - namespace: 11 bytes - token: <authentication_token> - ``` - - 1. Locate the API server endpoint so GitLab can connect to the cluster. This is displayed on - the AWS EKS console, when viewing the EKS cluster details. -1. Navigate to your: - - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. - - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. -1. Click **Add Kubernetes cluster**. -1. Click the **Add existing cluster** tab and fill in the details: - - **Kubernetes cluster name**: A name for the cluster to identify it within GitLab. - - **Environment scope**: Leave this as `*` for now, since we are only connecting a single cluster. - - **API URL**: The API server endpoint retrieved earlier. - - **CA Certificate**: The certificate data from the earlier step, as-is. - - **Service Token**: The admin token value. - - For project-level clusters, **Project namespace prefix**: This can be left blank to accept the - default namespace, based on the project name. -1. Click on **Add Kubernetes cluster**. The cluster is now connected to GitLab. - -At this point, [Kubernetes deployment variables](index.md#deployment-variables) will -automatically be available during CI/CD jobs, making it easy to interact with the cluster. - -If you would like to utilize your own CI/CD scripts to deploy to the cluster, you can stop here. +For information on adding an existing EKS cluster, see +[Existing Kubernetes cluster](add_remove_clusters.md#existing-kubernetes-cluster). ### Create a default Storage Class diff --git a/doc/user/project/clusters/add_gke_clusters.md b/doc/user/project/clusters/add_gke_clusters.md index 4094828323a..2746076befe 100644 --- a/doc/user/project/clusters/add_gke_clusters.md +++ b/doc/user/project/clusters/add_gke_clusters.md @@ -21,26 +21,25 @@ requirements are met: ## New GKE cluster -Starting from [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/25925), all the GKE clusters +Starting from [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/25925), all the GKE clusters provisioned by GitLab are [VPC-native](https://cloud.google.com/kubernetes-engine/docs/how-to/alias-ips). -### Important notes - Note the following: - The [Google authentication integration](../../../integration/google.md) must be enabled in GitLab at the instance level. If that's not the case, ask your GitLab administrator to enable it. On GitLab.com, this is enabled. -- Starting from [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/issues/55902), all GKE clusters +- Starting from [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55902), all GKE clusters created by GitLab are RBAC-enabled. Take a look at the [RBAC section](add_remove_clusters.md#rbac-cluster-resources) for more information. - Starting from [GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18341), the cluster's pod address IP range will be set to /16 instead of the regular /14. /16 is a CIDR notation. - GitLab requires basic authentication enabled and a client certificate issued for the cluster to - set up an [initial service account](add_remove_clusters.md#access-controls). Starting from [GitLab - 11.10](https://gitlab.com/gitlab-org/gitlab-foss/issues/58208), the cluster creation process will - explicitly request that basic authentication and client certificate is enabled. + set up an [initial service account](add_remove_clusters.md#access-controls). In [GitLab versions + 11.10 and later](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58208), the cluster creation process + explicitly requests GKE to create clusters with basic authentication enabled and a client + certificate. ### Creating the cluster on GKE diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index fddc9873f17..d0cba729e35 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -50,7 +50,7 @@ a `gitlab` service account with `cluster-admin` privileges is created in the `de to manage the newly created cluster. NOTE: **Note:** -Restricted service account for deployment was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/51716) in GitLab 11.5. +Restricted service account for deployment was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51716) in GitLab 11.5. When you install Helm into your cluster, the `tiller` service account is created with `cluster-admin` privileges in the `gitlab-managed-apps` @@ -151,146 +151,142 @@ For more information, see information for adding an: NOTE: **Note:** Kubernetes integration is not supported for arm64 clusters. See the issue -[Helm Tiller fails to install on arm64 cluster](https://gitlab.com/gitlab-org/gitlab-foss/issues/64044) for details. +[Helm Tiller fails to install on arm64 cluster](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64044) for details. ### Existing Kubernetes cluster To add a Kubernetes cluster to your project, group, or instance: 1. Navigate to your: - - Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. - - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - - **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. + 1. Project's **{cloud-gear}** **Operations > Kubernetes** page, for a project-level cluster. + 1. Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. + 1. **{admin}** **Admin Area >** **{cloud-gear}** **Kubernetes** page, for an instance-level cluster. 1. Click **Add Kubernetes cluster**. 1. Click the **Add existing cluster** tab and fill in the details: - - **Kubernetes cluster name** (required) - The name you wish to give the cluster. - - **Environment scope** (required) - The - [associated environment](index.md#setting-the-environment-scope-premium) to this cluster. - - **API URL** (required) - - It's the URL that GitLab uses to access the Kubernetes API. Kubernetes - exposes several APIs, we want the "base" URL that is common to all of them. - For example, `https://kubernetes.example.com` rather than `https://kubernetes.example.com/api/v1`. - - Get the API URL by running this command: - - ```shell - kubectl cluster-info | grep 'Kubernetes master' | awk '/http/ {print $NF}' - ``` - - - **CA certificate** (required) - A valid Kubernetes certificate is needed to authenticate to the cluster. We will use the certificate created by default. - - List the secrets with `kubectl get secrets`, and one should be named similar to - `default-token-xxxxx`. Copy that token name for use below. - - Get the certificate by running this command: - - ```shell - - kubectl get secret <secret name> -o jsonpath="{['data']['ca\.crt']}" | base64 --decode - - ``` - - NOTE: **Note:** - If the command returns the entire certificate chain, you need copy the *root ca* - certificate at the bottom of the chain. - - - **Token** - - GitLab authenticates against Kubernetes using service tokens, which are - scoped to a particular `namespace`. - **The token used should belong to a service account with - [`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) - privileges.** To create this service account: - - 1. Create a file called `gitlab-admin-service-account.yaml` with contents: - - ```yaml - apiVersion: v1 - kind: ServiceAccount - metadata: - name: gitlab-admin - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRoleBinding - metadata: - name: gitlab-admin - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin - subjects: - - kind: ServiceAccount - name: gitlab-admin - namespace: kube-system - ``` - - 1. Apply the service account and cluster role binding to your cluster: - - ```shell - kubectl apply -f gitlab-admin-service-account.yaml - ``` - - You will need the `container.clusterRoleBindings.create` permission - to create cluster-level roles. If you do not have this permission, - you can alternatively enable Basic Authentication and then run the - `kubectl apply` command as an admin: - - ```shell - kubectl apply -f gitlab-admin-service-account.yaml --username=admin --password=<password> - ``` - - NOTE: **Note:** - Basic Authentication can be turned on and the password credentials - can be obtained using the Google Cloud Console. - - Output: - - ```shell - serviceaccount "gitlab-admin" created - clusterrolebinding "gitlab-admin" created - ``` - - 1. Retrieve the token for the `gitlab-admin` service account: - - ```shell - kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep gitlab-admin | awk '{print $1}') - ``` - - Copy the `<authentication_token>` value from the output: - - ```yaml - Name: gitlab-admin-token-b5zv4 - Namespace: kube-system - Labels: <none> - Annotations: kubernetes.io/service-account.name=gitlab-admin + 1. **Kubernetes cluster name** (required) - The name you wish to give the cluster. + 1. **Environment scope** (required) - The + [associated environment](index.md#setting-the-environment-scope-premium) to this cluster. + 1. **API URL** (required) - + It's the URL that GitLab uses to access the Kubernetes API. Kubernetes + exposes several APIs, we want the "base" URL that is common to all of them. + For example, `https://kubernetes.example.com` rather than `https://kubernetes.example.com/api/v1`. + + Get the API URL by running this command: + + ```shell + kubectl cluster-info | grep 'Kubernetes master' | awk '/http/ {print $NF}' + ``` + + 1. **CA certificate** (required) - A valid Kubernetes certificate is needed to authenticate to the cluster. We will use the certificate created by default. + 1. List the secrets with `kubectl get secrets`, and one should be named similar to + `default-token-xxxxx`. Copy that token name for use below. + 1. Get the certificate by running this command: + + ```shell + kubectl get secret <secret name> -o jsonpath="{['data']['ca\.crt']}" | base64 --decode + ``` + + NOTE: **Note:** + If the command returns the entire certificate chain, you need copy the *root ca* + certificate at the bottom of the chain. + + 1. **Token** - + GitLab authenticates against Kubernetes using service tokens, which are + scoped to a particular `namespace`. + **The token used should belong to a service account with + [`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) + privileges.** To create this service account: + 1. Create a file called `gitlab-admin-service-account.yaml` with contents: + + ```yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: gitlab-admin + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: ClusterRoleBinding + metadata: + name: gitlab-admin + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: gitlab-admin + namespace: kube-system + ``` + + 1. Apply the service account and cluster role binding to your cluster: + + ```shell + kubectl apply -f gitlab-admin-service-account.yaml + ``` + + You will need the `container.clusterRoleBindings.create` permission + to create cluster-level roles. If you do not have this permission, + you can alternatively enable Basic Authentication and then run the + `kubectl apply` command as an admin: + + ```shell + kubectl apply -f gitlab-admin-service-account.yaml --username=admin --password=<password> + ``` + + NOTE: **Note:** + Basic Authentication can be turned on and the password credentials + can be obtained using the Google Cloud Console. + + Output: + + ```shell + serviceaccount "gitlab-admin" created + clusterrolebinding "gitlab-admin" created + ``` + + 1. Retrieve the token for the `gitlab-admin` service account: + + ```shell + kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep gitlab-admin | awk '{print $1}') + ``` + + Copy the `<authentication_token>` value from the output: + + ```yaml + Name: gitlab-admin-token-b5zv4 + Namespace: kube-system + Labels: <none> + Annotations: kubernetes.io/service-account.name=gitlab-admin kubernetes.io/service-account.uid=bcfe66ac-39be-11e8-97e8-026dce96b6e8 - Type: kubernetes.io/service-account-token - - Data - ==== - ca.crt: 1025 bytes - namespace: 11 bytes - token: <authentication_token> - ``` - - NOTE: **Note:** - For GKE clusters, you will need the - `container.clusterRoleBindings.create` permission to create a cluster - role binding. You can follow the [Google Cloud - documentation](https://cloud.google.com/iam/docs/granting-changing-revoking-access) - to grant access. - - - **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. - See the [Managed clusters section](index.md#gitlab-managed-clusters) for more information. - - - **Project namespace** (optional) - You don't have to fill it in; by leaving - it blank, GitLab will create one for you. Also: - - Each project should have a unique namespace. - - The project namespace is not necessarily the namespace of the secret, if - you're using a secret with broader permissions, like the secret from `default`. - - You should **not** use `default` as the project namespace. - - If you or someone created a secret specifically for the project, usually - with limited permissions, the secret's namespace and project namespace may - be the same. + Type: kubernetes.io/service-account-token + + Data + ==== + ca.crt: 1025 bytes + namespace: 11 bytes + token: <authentication_token> + ``` + + NOTE: **Note:** + For GKE clusters, you will need the + `container.clusterRoleBindings.create` permission to create a cluster + role binding. You can follow the [Google Cloud + documentation](https://cloud.google.com/iam/docs/granting-changing-revoking-access) + to grant access. + + 1. **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. + See the [Managed clusters section](index.md#gitlab-managed-clusters) for more information. + 1. **Project namespace** (optional) - You don't have to fill it in; by leaving + it blank, GitLab will create one for you. Also: + - Each project should have a unique namespace. + - The project namespace is not necessarily the namespace of the secret, if + you're using a secret with broader permissions, like the secret from `default`. + - You should **not** use `default` as the project namespace. + - If you or someone created a secret specifically for the project, usually + with limited permissions, the secret's namespace and project namespace may + be the same. 1. Finally, click the **Create Kubernetes cluster** button. @@ -337,7 +333,7 @@ To disable the Kubernetes cluster integration, follow the same procedure. To remove the Kubernetes cluster integration from your project, either: - Select **Remove integration**, to remove only the Kubernetes integration. -- [From GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/26815), select +- [From GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/26815), select **Remove integration and resources**, to also remove all related GitLab cluster resources (for example, namespaces, roles, and bindings) when removing the integration. diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 1298a24fcac..961a9fda5ff 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -6,10 +6,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Kubernetes clusters -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/35954) in GitLab 10.1 for projects. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/34758) in +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/35954) in GitLab 10.1 for projects. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34758) in > GitLab 11.6 for [groups](../../group/clusters/index.md). -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/39840) in +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39840) in > GitLab 11.11 for [instances](../../instance/clusters/index.md). GitLab provides many features with a Kubernetes integration. Kubernetes can be @@ -46,6 +46,7 @@ GitLab is committed to support at least two production-ready Kubernetes minor ve Currently, GitLab supports the following Kubernetes versions: +- 1.16 - 1.15 - 1.14 - 1.13 (deprecated, support ends on November 22, 2020) @@ -171,7 +172,7 @@ Note the following with GitLab and clusters: #### Clearing the cluster cache -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31759) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31759) in GitLab 12.6. If you choose to allow GitLab to manage your cluster for you, GitLab stores a cached version of the namespaces and service accounts it creates for your projects. If you @@ -314,7 +315,7 @@ If your cluster was created before GitLab 12.2, default `KUBE_NAMESPACE` will be ### Custom namespace -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27630) in GitLab 12.6. The Kubernetes integration defaults to project-environment-specific namespaces of the form `<project_name>-<project_id>-<environment>` (see [Deployment @@ -326,7 +327,7 @@ in `.gitlab-ci.yml`. NOTE: **Note:** When using a [GitLab-managed cluster](#gitlab-managed-clusters), the namespaces are created automatically prior to deployment and [can not be -customized](https://gitlab.com/gitlab-org/gitlab/issues/38054). +customized](https://gitlab.com/gitlab-org/gitlab/-/issues/38054). ### Troubleshooting diff --git a/doc/user/project/clusters/kubernetes_pod_logs.md b/doc/user/project/clusters/kubernetes_pod_logs.md index 8577231b69b..509be4ed0a8 100644 --- a/doc/user/project/clusters/kubernetes_pod_logs.md +++ b/doc/user/project/clusters/kubernetes_pod_logs.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Kubernetes Logs -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4752) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4752) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26383) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9. GitLab makes it easy to view the logs of running pods in [connected Kubernetes clusters](index.md). @@ -65,23 +65,23 @@ Logs can be displayed by clicking on a specific pod from [Deploy Boards](../depl The **Log Explorer** lets you filter the logs by: - Pods. -- [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/5769), environments. +- [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/5769), environments. - [From GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21656), [full text search](#full-text-search). -- [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/issues/197879), dates. +- [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/197879), dates. Loading more than 500 log lines is possible from [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/198050) onward. Support for pods with multiple containers is coming -[in a future release](https://gitlab.com/gitlab-org/gitlab/issues/13404). +[in a future release](https://gitlab.com/gitlab-org/gitlab/-/issues/13404). Support for historical data is coming -[in a future release](https://gitlab.com/gitlab-org/gitlab/issues/196191). +[in a future release](https://gitlab.com/gitlab-org/gitlab/-/issues/196191). ### Filter by date -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197879) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/197879) in GitLab 12.8. When you enable [Elastic Stack](../../clusters/applications.md#elastic-stack) on your cluster, you can filter logs displayed in the **Log Explorer** by date. diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index dfed43470bc..92ef35ad93f 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -23,7 +23,7 @@ pre-written code blocks or database queries against a given environment. ## Executable Runbooks -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/45912) in GitLab 11.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45912) in GitLab 11.4. The JupyterHub app offered via GitLab’s Kubernetes integration now ships with Nurtch’s Rubix library, providing a simple way to create DevOps diff --git a/doc/user/project/clusters/serverless/aws.md b/doc/user/project/clusters/serverless/aws.md index 124a0d4bf9f..15f7e14fda9 100644 --- a/doc/user/project/clusters/serverless/aws.md +++ b/doc/user/project/clusters/serverless/aws.md @@ -15,7 +15,7 @@ GitLab supports deployment of AWS Lambda functions through GitLab CI/CD using th ## Serverless Framework -The [Serverless Framework can deploy to AWS](https://serverless.com/framework/docs/providers/aws/). +The [Serverless Framework can deploy to AWS](https://www.serverless.com/framework/docs/providers/aws/). We have prepared an example with a step-by-step guide to create a simple function and deploy it on AWS. @@ -101,7 +101,7 @@ The handler definition will provision the Lambda function using the source code The `events` declaration will create a AWS API Gateway `GET` endpoint to receive external requests and hand them over to the Lambda function via a service integration. -You can read more about the available properties and additional configuration possibilities of the Serverless Framework here: <https://serverless.com/framework/docs/providers/aws/guide/serverless.yml/> +You can read more about the [available properties and additional configuration possibilities](https://www.serverless.com/framework/docs/providers/aws/guide/serverless.yml/) of the Serverless Framework. #### Crafting the `.gitlab-ci.yml` file @@ -134,7 +134,7 @@ This example code does the following: #### Setting up your AWS credentials with your GitLab account In order to interact with your AWS account, the GitLab CI/CD pipelines require both `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to be defined in your GitLab settings under **Settings > CI/CD > Variables**. -For more information please see: <https://docs.gitlab.com/ee/ci/variables/README.html#via-the-ui> +For more information please see [Create a custom variable in the UI](../../../../ci/variables/README.md#create-a-custom-variable-in-the-ui). NOTE: **Note:** The AWS credentials you provide must include IAM policies that provision correct access control to AWS Lambda, API Gateway, CloudFormation, and IAM resources. @@ -275,7 +275,7 @@ module.exports.hello = async event => { }; ``` -For more information, see the [Your CORS and API Gateway survival guide](https://serverless.com/blog/cors-api-gateway-survival-guide/) +For more information, see the [Your CORS and API Gateway survival guide](https://www.serverless.com/blog/cors-api-gateway-survival-guide/) blog post written by the Serverless Framework team. #### Writing automated tests @@ -288,7 +288,7 @@ automated testing of both local and deployed serverless function. The example code is available: -- As a [cloneable repository](https://gitlab.com/gitlab-org/serverless/examples/serverless-framework-js). +- As a [clonable repository](https://gitlab.com/gitlab-org/serverless/examples/serverless-framework-js). - In a version with [tests and secret variables](https://gitlab.com/gitlab-org/project-templates/serverless-framework/). You can also use a [template](../../../../gitlab-basics/create-project.md#project-templates) @@ -416,7 +416,7 @@ production: environment: production ``` -Let’s examine the config file more closely: +Let’s examine the configuration file more closely: - `image` specifies the Docker image to use for this build. This is the latest Python image since the sample application is written in Python. diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index 2156d96f92a..45fb313d177 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -33,7 +33,7 @@ of the box through its main components: - [Serving](https://github.com/knative/serving): Request-driven compute that can scale to zero. - [Eventing](https://github.com/knative/eventing): Management and delivery of events. -For more information on Knative, visit the [Knative docs repo](https://github.com/knative/docs). +For more information on Knative, visit the [Knative docs repository](https://github.com/knative/docs). With GitLab Serverless, you can deploy both functions-as-a-service (FaaS) and serverless applications. @@ -61,14 +61,14 @@ To run Knative on GitLab, you will need: wildcard domain where your applications will be served. Configure your DNS server to use the external IP address or hostname for that domain. 1. **`.gitlab-ci.yml`:** GitLab uses [Kaniko](https://github.com/GoogleContainerTools/kaniko) - to build the application. We also use [gitlabktl](https://gitlab.com/gitlab-org/gitlabktl) + to build the application. We also use [GitLab Knative tool](https://gitlab.com/gitlab-org/gitlabktl) CLI to simplify the deployment of services and functions to Knative. 1. **`serverless.yml`** (for [functions only](#deploying-functions)): When using serverless to deploy functions, the `serverless.yml` file will contain the information for all the functions being hosted in the repository as well as a reference to the runtime being used. 1. **`Dockerfile`** (for [applications only](#deploying-serverless-applications)): Knative requires a `Dockerfile` in order to build your applications. It should be included at the root of your - project's repo and expose port `8080`. `Dockerfile` is not require if you plan to build serverless functions + project's repository and expose port `8080`. `Dockerfile` is not require if you plan to build serverless functions using our [runtimes](https://gitlab.com/gitlab-org/serverless/runtimes). 1. **Prometheus** (optional): Installing Prometheus allows you to monitor the scale and traffic of your serverless function/application. See [Installing Applications](../index.md#installing-applications) for more information. @@ -97,9 +97,9 @@ The minimum recommended cluster size to run Knative is 3-nodes, 6 vCPUs, and 22. 1. The Ingress is now available at this address and will route incoming requests to the proper service based on the DNS name in the request. To support this, a wildcard DNS record should be created for the desired domain name. For example, if your Knative base domain is `knative.info` then you need to create an A record or CNAME record with domain `*.knative.info` - pointing the ip address or hostname of the Ingress. + pointing the IP address or hostname of the Ingress. -  +  NOTE: **Note:** You can deploy either [functions](#deploying-functions) or [serverless applications](#deploying-serverless-applications) @@ -107,7 +107,7 @@ on a given project but not both. The current implementation makes use of a `serv ## Using an existing installation of Knative -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/58941) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58941) in GitLab 12.0. NOTE: **Note:** The "invocations" monitoring feature of GitLab serverless will not work when @@ -199,7 +199,7 @@ You must provide a `Dockerfile` to run serverless functions if no runtime is spe ### OpenFaaS runtimes -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29253) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29253) in GitLab 12.5. [OpenFaaS classic runtimes](https://github.com/openfaas/templates#templates-in-store) can be used with GitLab serverless. @@ -318,7 +318,7 @@ Explanation of the fields used above: |-----------|-------------| | `name` | Indicates which provider is used to execute the `serverless.yml` file. In this case, the TriggerMesh middleware. | | `envs` | Includes the environment variables to be passed as part of function execution for **all** functions in the file, where `FOO` is the variable name and `BAR` are the variable contents. You may replace this with your own variables. | -| `secrets` | Includes the contents of the Kubernetes secret as environment variables accessible to be passed as part of function execution for **all** functions in the file. The secrets are expected in ini format. | +| `secrets` | Includes the contents of the Kubernetes secret as environment variables accessible to be passed as part of function execution for **all** functions in the file. The secrets are expected in INI format. | ### `functions` @@ -332,7 +332,7 @@ subsequent lines contain the function attributes. | `runtime` (optional)| The runtime to be used to execute the function. This can be a runtime alias (see [Runtime aliases](#runtime-aliases)), or it can be a full URL to a custom runtime repository. When the runtime is not specified, we assume that `Dockerfile` is present in the function directory specified by `source`. | | `description` | A short description of the function. | | `envs` | Sets an environment variable for the specific function only. | -| `secrets` | Includes the contents of the Kubernetes secret as environment variables accessible to be passed as part of function execution for the specific function only. The secrets are expected in ini format. | +| `secrets` | Includes the contents of the Kubernetes secret as environment variables accessible to be passed as part of function execution for the specific function only. The secrets are expected in INI format. | ### Deployment @@ -384,7 +384,7 @@ The sample function can now be triggered from any HTTP client using a simple `PO http://functions-echo.functions-1.functions.example.com/ ``` - 1. Using a web-based tool (ie. postman, restlet, etc) + 1. Using a web-based tool (such as Postman or Restlet)  @@ -516,7 +516,7 @@ cluster. ## Configuring logging -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/33330) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33330) in GitLab 12.5. ### Prerequisites diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 40ea1833fa3..6b81aea4b87 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -6,8 +6,8 @@ type: reference > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6916) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.3. -> - [Support for group namespaces](https://gitlab.com/gitlab-org/gitlab-foss/issues/53182) added in GitLab Starter 12.1. -> - Code Owners for Merge Request approvals was [introduced](https://gitlab.com/gitlab-org/gitlab/issues/4418) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.9. +> - [Support for group namespaces](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53182) added in GitLab Starter 12.1. +> - Code Owners for Merge Request approvals was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4418) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.9. ## Introduction @@ -73,12 +73,28 @@ be used for merge request approvals: - As [merge request eligible approvers](merge_requests/merge_request_approvals.md#code-owners-as-eligible-approvers). - As required approvers for [protected branches](protected_branches.md#protected-branches-approval-by-code-owners-premium). **(PREMIUM)** +NOTE: **Note**: +Developer or higher [permissions](../permissions.md) are required in order to +approve a merge request. + Once set, Code Owners are displayed in merge requests widgets:  -NOTE: **Note**: - While the`CODEOWNERS` file can be used in addition to Merge Request [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules) it can also be used as the sole driver of a Merge Request approval (without using [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules)) by simply creating the file in one of the three locations specified above, configuring the Code Owners to be required approvers for [protected branches](protected_branches.md#protected-branches-approval-by-code-owners-premium) and then using [the syntax of Code Owners files](code_owners.md#the-syntax-of-code-owners-files) to specify the actual owners and granular permissions. +While the `CODEOWNERS` file can be used in addition to Merge Request [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules) +it can also be used as the sole driver of merge request approvals +(without using [Approval Rules](merge_requests/merge_request_approvals.md#approval-rules)). +To do so, create the file in one of the three locations specified above and +set the code owners as required approvers for [protected branches](protected_branches.md#protected-branches-approval-by-code-owners-premium). +Use [the syntax of Code Owners files](code_owners.md#the-syntax-of-code-owners-files) +to specify the actual owners and granular permissions. + +Using Code Owners in conjunction with [Protected Branches Approvals](protected_branches.md#protected-branches-approval-by-code-owners-premium) +will prevent any user who is not specified in the `CODEOWNERS` file from pushing changes +for the specified files/paths, even if their role is included in the **Allowed to push** column. +This allows for a more inclusive push strategy, as administrators don't have to restrict developers +from pushing directly to the protected branch, but can restrict pushing to certain +files where a review by Code Owners is required. ## The syntax of Code Owners files @@ -93,7 +109,7 @@ groups or subgroups from the project's group hierarchy as potential code owners. For example, consider the following hierarchy for a given project: -```text +```plaintext group >> sub-group >> sub-subgroup >> myproject >> file.md ``` @@ -103,7 +119,7 @@ Any of the following groups would be eligible to be specified as code owners: - `@group/sub-group` - `@group/sub-group/sub-subgroup` -In addition, any groups that have been invited to the project using the **Settings > Members** tool will also be recognized as eligible code owners. +In addition, any groups that have been invited to the project using the **Members** tool will also be recognized as eligible code owners. The order in which the paths are defined is significant: the last pattern that matches a given path will be used to find the code diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index 8f7bb844e37..0a613ff918b 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -1,6 +1,13 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto, reference +--- + # Deploy Boards **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1589) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1589) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. GitLab's Deploy Boards offer a consolidated view of the current health and status of each CI [environment](../../ci/environments/index.md) running on [Kubernetes](https://kubernetes.io), displaying the status @@ -34,6 +41,10 @@ knowledge. In particular, you should be familiar with: - [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) - [Kubernetes canary deployments](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments) +NOTE: **Note:** +Apps that consist of multiple deployments are shown as duplicates on the deploy board. +Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/8463) for details. + ## Use cases Since the Deploy Board is a visual representation of the Kubernetes pods for a @@ -68,7 +79,7 @@ To display the Deploy Boards for a specific [environment](../../ci/environments/ instead of `DeploymentConfiguration`. Otherwise, the Deploy Boards won't render correctly. For more information, read the [OpenShift docs](https://docs.openshift.com/container-platform/3.7/dev_guide/deployments/kubernetes_deployments.html#kubernetes-deployments-vs-deployment-configurations) - and [GitLab issue #4584](https://gitlab.com/gitlab-org/gitlab/issues/4584). + and [GitLab issue #4584](https://gitlab.com/gitlab-org/gitlab/-/issues/4584). 1. [Configure GitLab Runner](../../ci/runners/README.md) with the [Docker](https://docs.gitlab.com/runner/executors/docker.html) or [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html) executor. diff --git a/doc/user/project/deploy_keys/img/deploy_keys_v13_0.png b/doc/user/project/deploy_keys/img/deploy_keys_v13_0.png Binary files differnew file mode 100644 index 00000000000..462141ef82a --- /dev/null +++ b/doc/user/project/deploy_keys/img/deploy_keys_v13_0.png diff --git a/doc/user/project/deploy_keys/img/public_deploy_key_v13_0.png b/doc/user/project/deploy_keys/img/public_deploy_key_v13_0.png Binary files differnew file mode 100644 index 00000000000..3e6d1605f95 --- /dev/null +++ b/doc/user/project/deploy_keys/img/public_deploy_key_v13_0.png diff --git a/doc/user/project/deploy_keys/index.md b/doc/user/project/deploy_keys/index.md new file mode 100644 index 00000000000..32d3eab5e62 --- /dev/null +++ b/doc/user/project/deploy_keys/index.md @@ -0,0 +1,164 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto, reference +--- + +# Deploy Keys + +Deploy keys allow read-only or read-write (if enabled) access to one or +more repositories, by importing an SSH public key to your GitLab instance. + +This is useful for cloning repositories to your Continuous +Integration (CI) server. By using deploy keys, you don't have to set up a +dummy user account. + +There are two types of deploy keys: + +- [Project deploy keys](#project-deploy-keys) +- [Public deploy keys](#public-deploy-keys) + +## Key details on deploy keys + +Deploy Keys allow a remote machine (VM, physical, and so on) to access a GitLab +repository with just a few steps. If you want a remote machine to interact with a GitLab +repository in automation, it's a simple solution. + +A drawback is that your repository could become vulnerable if a remote machine is compromised +by a hacker. You should limit access to the remote machine before a deploy key is +enabled on your repository. A good rule to follow is to access only to trusted users, +and make sure that the allowed users have [maintainer permissions or higher](../../permissions.md) +in the GitLab project. + +If this security implication is a concern for your organization, +[Deploy Tokens](../deploy_tokens/index.md) works as an alternative, but with more +security control. + +## Deploy Keys Permissions + +You can choose the access level of a deploy key when you enable it on a project: + +- `read-only`: The deploy key can read a repository. +- `read-write`: The deploy key can read a repository and write to it. + +Project maintainers and owners can activate and deactivate deploy keys. +They can also add their own deploy keys and enable them for this project. + +When a `write-access` deploy key is used to push a commit, GitLab checks if +the **creator** of the deploy key has permission to access the resource. For example: + +- When a deploy key is used to push a commit to a [protected branch](../protected_branches.md), + the **creator** of the deploy key must have access to the branch. +- When a deploy key is used to push a commit that triggers a CI/CD pipelines, the **creator** of + the deploy key must have access to the CI/CD resources (like protected environments, secret variables, and so on). +- If the **creator** of a deploy key does not have permissions to read a project's + repository, the deploy key _might_ encounter an error during the process. + +## Differences between deploy keys and deploy tokens + +Both deploy keys and [deploy tokens](../deploy_tokens/index.md#deploy-tokens) can +help you access a repository, but there are some notables differences between them: + +- Deploy keys are shareable between projects that are not related or don't even + belong to the same group. Deploy tokens belong to either a project or + [a group](../deploy_tokens/index.md#group-deploy-token). +- A deploy key is an SSH key you need to generate yourself on your machine. A deploy + token is generated by your GitLab instance, and is provided to users only once + (at creation time). +- A deploy key is valid as long as it's registered and enabled. Deploy tokens can + be time-sensitive, as you can control their validity by setting an expiration date to them. +- You can't log in to a registry with deploy keys, or perform read / write operations + on it, but this [is possible with deploy tokens](../deploy_tokens/index.md#gitlab-deploy-token). +- You need an SSH key pair to use deploy keys, but not deploy tokens. + +## How to enable Deploy Keys + +### Project deploy keys + +[Project maintainers and owners](../../permissions.md#project-members-permissions) +can add or enable a deploy key for a project repository: + +1. Navigate to the project's **Settings > Repository** page. +1. Expand the **Deploy Keys** section. +1. Specify a title for the new deploy key and paste your public SSH key. +1. (Optional) Check **Write access allowed** to allow `read-write` access. Leave it unchecked for `read-only` access. + +There are three lists of Project Deploy Keys: + +- Enabled deploy keys +- Privately accessible deploy keys +- Public accessible deploy keys + + + +After you add a key, it will be enabled for this project by default, and it'll appear +in the **Enabled deploy keys** tab. + +In the **Privately accessible deploy keys** tab, you can enable a private key which +has been already imported in a different project. If you have access to these keys, +it's because you have either: + +- Previously uploaded the keys yourself in a different project. +- You are a maintainer or owner of the other project where the keys were imported. + +In the **Publicly accessible deploy keys** tab, you can enable +keys that were [made available to your entire GitLab instance](#public-deploy-keys). + +After a key is added, you can edit it to update its title, or switch between `read-only` +and `read-write` access. + +NOTE: **Note:** +If you have enabled a privately or publicly accessible or deploy key for your +project, and if you then update the access level for this key from `read-only` to +`read-write`, the change will be only for the **current project**. + +### Public deploy keys + +Public deploy keys allow `read-only` or `read-write` +access to any repository in your GitLab instance. This is useful for integrating +repositories to secure, shared services, such as CI/CD. + +Instance administrators can add public deploy keys: + +1. Go to **Admin Area** (**{admin}**) **> Deploy Keys**. +1. Click on **New deploy key**. + + Make sure your new key has a meaningful title, as it is the primary way for project + maintainers and owners to identify the correct public deploy key to add. For example, + if the key gives access to a SaaS CI/CD instance, use the name of that service + in the key name if that is all the key is used for. + + + +After adding a key, it will be available to any shared systems. Project maintainers +or higher can [authorize a public deploy key](#project-deploy-keys) to start using it with the project. + +NOTE: **Note:** +The **Publicly accessible deploy keys** tab within Project's CI/CD settings only appears +if there is at least one Public deploy key configured. + +Public deploy keys can provide greater security compared to project deploy keys, as +the administrator of the target integrated system is the only one who needs to know the key value, +or configure it. + +When creating a Public deploy key, determine whether or not it can be defined for +very narrow usage, such as just a specific service, or if it needs to be defined for +broader usage, such as full `read-write` access for all services. + +CAUTION: **Warning:** +Adding a public deploy key does not immediately expose any repository to it. Public +deploy keys enable access from other systems, but access is not given to any project +until a project maintainer chooses to make use of it. + +## Troubleshooting + +### Deploy Key cannot push to a protected branch + +If the owner of this deploy key does not have access to a [protected +branch](../protected_branches.md), then this deploy key won't have access to +the branch either. In addition to this, choosing the **No one** value in +[the "Allowed to push" section](../protected_branches.md#configuring-protected-branches) +means that no users **and** no services using deploy keys can push to that selected branch. + +Refer to [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/30769) for more information. diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index 2d42debed68..10f0281d0eb 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -1,7 +1,14 @@ +--- +stage: Release +group: Progressive Delivery +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + # Deploy Tokens > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17894) in GitLab 10.7. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/199370) from **Settings > Repository** in GitLab 12.9. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/199370) from **Settings > Repository** in GitLab 12.9. > - [Added `write_registry` scope](https://gitlab.com/gitlab-org/gitlab/-/issues/22743) in GitLab 12.10. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/29280) from **Settings > CI / CD** in GitLab 12.10.1. > - [Added package registry scopes](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) from **Settings > CI / CD** in GitLab 13.0. @@ -120,7 +127,7 @@ To upload packages in the GitLab package registry, you'll need to: ### Group Deploy Token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/21765) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21765) in GitLab 12.9. A deploy token created at the group level can be used across all projects that belong either to the specific group or to one of its subgroups. diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index 16ac53a2b52..0f90c321a14 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Description templates >[Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4981) in GitLab 8.11. @@ -29,7 +35,7 @@ templates of the default branch will be taken into account. For example, if you have a project for tracking new blog posts, you can require the title, outlines, author name, author social media information, and so on. - Following the previous example, you can make a template for every MR submitted - with a new blog post, requiring information about the post date, frontmatter data, + with a new blog post, requiring information about the post date, front matter data, images guidelines, link to the related issue, reviewer name, and so on. - You can also create issues and merge request templates for different stages of your workflow, for example, feature proposal, feature improvement, or a bug report. diff --git a/doc/user/project/img/status_page_detail_link_v13_1.png b/doc/user/project/img/status_page_detail_link_v13_1.png Binary files differnew file mode 100644 index 00000000000..f3d1005447c --- /dev/null +++ b/doc/user/project/img/status_page_detail_link_v13_1.png diff --git a/doc/user/project/import/bitbucket.md b/doc/user/project/import/bitbucket.md index 56717858b53..56266718d12 100644 --- a/doc/user/project/import/bitbucket.md +++ b/doc/user/project/import/bitbucket.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import your project from Bitbucket Cloud to GitLab NOTE: **Note:** diff --git a/doc/user/project/import/bitbucket_server.md b/doc/user/project/import/bitbucket_server.md index 55df2d7294d..f0e730564d8 100644 --- a/doc/user/project/import/bitbucket_server.md +++ b/doc/user/project/import/bitbucket_server.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import your project from Bitbucket Server to GitLab > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20164) in GitLab 11.2. diff --git a/doc/user/project/import/clearcase.md b/doc/user/project/import/clearcase.md index 89a9f7da852..173ba71b167 100644 --- a/doc/user/project/import/clearcase.md +++ b/doc/user/project/import/clearcase.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Migrating from ClearCase [ClearCase](https://www.ibm.com/us-en/marketplace/rational-clearcase) is a set of diff --git a/doc/user/project/import/cvs.md b/doc/user/project/import/cvs.md index 3b2404912f7..d2e79458526 100644 --- a/doc/user/project/import/cvs.md +++ b/doc/user/project/import/cvs.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Migrating from CVS [CVS](https://savannah.nongnu.org/projects/cvs) is an old centralized version diff --git a/doc/user/project/import/fogbugz.md b/doc/user/project/import/fogbugz.md index 13409c93929..149b5d1913c 100644 --- a/doc/user/project/import/fogbugz.md +++ b/doc/user/project/import/fogbugz.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import your project from FogBugz to GitLab It only takes a few simple steps to import your project from FogBugz. diff --git a/doc/user/project/import/gemnasium.md b/doc/user/project/import/gemnasium.md index cf9ac15f5ac..0d6e059f1cf 100644 --- a/doc/user/project/import/gemnasium.md +++ b/doc/user/project/import/gemnasium.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Gemnasium **(ULTIMATE)** This guide describes how to migrate from Gemnasium.com to your own GitLab diff --git a/doc/user/project/import/gitea.md b/doc/user/project/import/gitea.md index 94ab9d9195b..543fffd33d6 100644 --- a/doc/user/project/import/gitea.md +++ b/doc/user/project/import/gitea.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import your project from Gitea to GitLab Import your projects from Gitea to GitLab with minimal effort. diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 4c213f21920..b7754e60837 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import your project from GitHub to GitLab Using the importer, you can import your GitHub repositories to GitLab.com or to diff --git a/doc/user/project/import/gitlab_com.md b/doc/user/project/import/gitlab_com.md index 2f87f257754..b9aea629e85 100644 --- a/doc/user/project/import/gitlab_com.md +++ b/doc/user/project/import/gitlab_com.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Project importing from GitLab.com to your private GitLab instance You can import your existing GitLab.com projects to your GitLab instance, but keep in diff --git a/doc/user/project/import/index.md b/doc/user/project/import/index.md index a0da68eb513..86b671c8371 100644 --- a/doc/user/project/import/index.md +++ b/doc/user/project/import/index.md @@ -1,3 +1,10 @@ +--- +type: reference, howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Migrating projects to a GitLab instance 1. [From Bitbucket Cloud](bitbucket.md) @@ -47,6 +54,10 @@ Keep in mind the limitations of the [import/export feature](../settings/import_e You will still need to migrate your Container Registry over a series of Docker pulls and pushes and re-run any CI pipelines to retrieve any build artifacts. +## Migrating from GitLab.com to self-managed GitLab + +The process is essentially the same as for [migrating from self-managed GitLab to GitLab.com](#migrating-from-self-managed-gitlab-to-gitlabcom). The main difference is that users can be created on the self-managed GitLab instance by an admin through the UI or the [users API](../../../api/users.md#user-creation). + ## Migrating between two self-managed GitLab instances The best method for migrating from one GitLab instance to another, diff --git a/doc/user/project/import/jira.md b/doc/user/project/import/jira.md index db48282a8f3..0b8807bb9b3 100644 --- a/doc/user/project/import/jira.md +++ b/doc/user/project/import/jira.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import your Jira project issues to GitLab > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2766) in GitLab 12.10. @@ -43,6 +49,7 @@ Importing large projects may take several minutes depending on the size of the i 1. On the **{issues}** **Issues** page, click the **Import Issues** (**{import}**) button. 1. Select **Import from Jira**. + This option is only visible if you have the [correct permissions](#permissions).  diff --git a/doc/user/project/import/manifest.md b/doc/user/project/import/manifest.md index 86c8b276887..0374e0acf9a 100644 --- a/doc/user/project/import/manifest.md +++ b/doc/user/project/import/manifest.md @@ -1,6 +1,13 @@ +--- +type: howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import multiple repositories by uploading a manifest file -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28811) in GitLab 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28811) in GitLab 11.2. GitLab allows you to import all the required Git repositories based on a manifest file like the one used by the diff --git a/doc/user/project/import/perforce.md b/doc/user/project/import/perforce.md index cbcef7a2fb0..dbc1c491493 100644 --- a/doc/user/project/import/perforce.md +++ b/doc/user/project/import/perforce.md @@ -1,3 +1,10 @@ +--- +type: howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Migrating from Perforce Helix [Perforce Helix](https://www.perforce.com/) provides a set of tools which also diff --git a/doc/user/project/import/phabricator.md b/doc/user/project/import/phabricator.md index c81d489f12b..a19068199db 100644 --- a/doc/user/project/import/phabricator.md +++ b/doc/user/project/import/phabricator.md @@ -1,6 +1,13 @@ +--- +type: howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import Phabricator tasks into a GitLab project -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/60562) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/60562) in GitLab 12.0. GitLab allows you to import all tasks from a Phabricator instance into GitLab issues. The import creates a single project with the diff --git a/doc/user/project/import/repo_by_url.md b/doc/user/project/import/repo_by_url.md index c20b1cb7f5e..9b5e43aae79 100644 --- a/doc/user/project/import/repo_by_url.md +++ b/doc/user/project/import/repo_by_url.md @@ -1,3 +1,10 @@ +--- +type: howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Import project from repo by URL You can import your existing repositories by providing the Git URL: diff --git a/doc/user/project/import/svn.md b/doc/user/project/import/svn.md index cf034aafca9..d5f4a014705 100644 --- a/doc/user/project/import/svn.md +++ b/doc/user/project/import/svn.md @@ -1,3 +1,10 @@ +--- +type: howto +stage: Manage +group: Import +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Migrating from SVN to GitLab Subversion (SVN) is a central version control system (VCS) while @@ -75,7 +82,7 @@ For more information regarding the SubGit configuration options, refer to ### Initial translation -Now that SubGit has configured the Git/SVN repos, run `subgit` to perform the +Now that SubGit has configured the Git/SVN repositories, run `subgit` to perform the initial translation of existing SVN revisions into the Git repository: ```shell @@ -159,7 +166,7 @@ svn2git https://svn.example.com/path/to/repo --authors /path/to/authors.txt ``` If your SVN repository requires a username and password add the -`--username <username>` and `--password <password` flags to the above command. +`--username <username>` and `--password <password>` flags to the above command. `svn2git` also supports excluding certain file paths, branches, tags, etc. See [svn2git documentation](https://github.com/nirvdrum/svn2git) or run `svn2git --help` for full documentation on all of the available options. diff --git a/doc/user/project/import/tfvc.md b/doc/user/project/import/tfvc.md index 9b148224e10..1c9ee7476bd 100644 --- a/doc/user/project/import/tfvc.md +++ b/doc/user/project/import/tfvc.md @@ -6,7 +6,7 @@ type: concepts Team Foundation Server (TFS), renamed [Azure DevOps Server](https://azure.microsoft.com/en-us/services/devops/server/) in 2019, is a set of tools developed by Microsoft which also includes -[Team Foundation Version Control](https://docs.microsoft.com/en-us/azure/devops/repos/tfvc/overview?view=azure-devops) +[Team Foundation Version Control](https://docs.microsoft.com/en-us/azure/devops/repos/tfvc/what-is-tfvc?view=azure-devops) (TFVC), a centralized version control system similar to Git. In this document, we focus on the TFVC to Git migration. diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 50272f0e007..3a4e240fb6c 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -100,7 +100,7 @@ When you create a project in GitLab, you'll have access to a large number of - [Maven packages](../packages/maven_repository/index.md): your private Maven repository in GitLab. **(PREMIUM)** - [NPM packages](../packages/npm_registry/index.md): your private NPM package registry in GitLab. **(PREMIUM)** - [Code owners](code_owners.md): specify code owners for certain files **(STARTER)** -- [License Compliance](../compliance/license_compliance/index.md): approve and blacklist licenses for projects. **(ULTIMATE)** +- [License Compliance](../compliance/license_compliance/index.md): approve and deny licenses for projects. **(ULTIMATE)** - [Dependency List](../application_security/dependency_list/index.md): view project dependencies. **(ULTIMATE)** - [Requirements](requirements/index.md): Requirements allow you to create criteria to check your products against. **(ULTIMATE)** - [Static Site Editor](static_site_editor/index.md): quickly edit content on static websites without prior knowledge of the codebase or Git commands. @@ -144,6 +144,17 @@ To view your starred projects: - Number of open merge requests - Number of open issues +### Explore projects + +You can explore other popular projects available on GitLab. To explore projects: + +1. Click **Projects** in the navigation bar. +1. Click **Explore Projects**. + +GitLab displays a list of projects, sorted by last updated date. To view +projects with the most [stars](#star-a-project), click **Most stars**. To view +projects with the largest number of comments in the past month, click **Trending**. + ## Project settings Set the project's visibility level and the access levels to its various pages @@ -250,14 +261,14 @@ password <personal_access_token> ## Access project page with project ID -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53671) in GitLab 11.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53671) in GitLab 11.8. To quickly access a project from the GitLab UI using the project ID, visit the `/projects/:id` URL in your browser or other tool accessing the project. ## Project aliases **(PREMIUM ONLY)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3264) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3264) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.1. When migrating repositories to GitLab and they are being accessed by other systems, it's very useful to be able to access them using the same name especially when diff --git a/doc/user/project/insights/index.md b/doc/user/project/insights/index.md index 52fcad8dd80..3cc76beb323 100644 --- a/doc/user/project/insights/index.md +++ b/doc/user/project/insights/index.md @@ -273,7 +273,7 @@ you defined. ### `projects` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10904) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10904) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. You can limit where the "issuables" can be queried from: diff --git a/doc/user/project/integrations/bamboo.md b/doc/user/project/integrations/bamboo.md index db8f24fc1e1..7b21c590c8a 100644 --- a/doc/user/project/integrations/bamboo.md +++ b/doc/user/project/integrations/bamboo.md @@ -22,7 +22,7 @@ need to be configured in a Bamboo build plan before GitLab can integrate. 1. Choose 'Repository triggers the build when changes are committed' 1. Check one or more repositories checkboxes 1. Enter the GitLab IP address in the 'Trigger IP addresses' box. This is a - whitelist of IP addresses that are allowed to trigger Bamboo builds. + list of IP addresses that are allowed to trigger Bamboo builds. 1. Save the trigger. 1. In the left pane, select a build stage. If you have multiple build stages you want to select the last stage that contains the Git checkout task. diff --git a/doc/user/project/integrations/custom_issue_tracker.md b/doc/user/project/integrations/custom_issue_tracker.md index 848e89c18cb..4eaf3a0d4b4 100644 --- a/doc/user/project/integrations/custom_issue_tracker.md +++ b/doc/user/project/integrations/custom_issue_tracker.md @@ -1,25 +1,34 @@ -# Custom Issue Tracker Service +# Custom Issue Tracker service -To enable the Custom Issue Tracker integration in a project, navigate to the -[Integrations page](overview.md#accessing-integrations), click -the **Customer Issue Tracker** service, and fill in the required details on the page as described -in the table below. You will be able to edit the title and description later as well. +To enable the Custom Issue Tracker integration in a project: -| Field | Description | -| ----- | ----------- | -| `title` | A title for the issue tracker (to differentiate between instances, for example). | -| `description` | A name for the issue tracker (to differentiate between instances, for example) | -| `project_url` | The URL to the project in the custom issue tracker. | -| `issues_url` | The URL to the issue in the issue tracker project that is linked to this GitLab project. Note that the `issues_url` requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. For example, `https://customissuetracker.com/project-name/:id`. | -| `new_issue_url` | Currently unused. Will be changed in a future release. | +1. Go to **{settings}** **Settings > Integrations**. +1. Click **Custom Issue Tracker** +1. Fill in the tracker's details, such as title, description, and URLs. + You will be able to edit these fields later as well. -Once you have configured and enabled Custom Issue Tracker Service you'll see a link on the GitLab project pages that takes you to that custom issue tracker. + These are some of the required fields: + + | Field | Description | + | --------------- | ----------- | + | **Title** | A title for the issue tracker (for example, to differentiate between instances). | + | **Description** | A name for the issue tracker (for example, to differentiate between instances). | + | **Project URL** | The URL to the project in the custom issue tracker. | + | **Issues URL** | The URL to the issue in the issue tracker project that is linked to this GitLab project. Note that the `issues_url` requires `:id` in the URL. This ID is used by GitLab as a placeholder to replace the issue number. For example, `https://customissuetracker.com/project-name/:id`. | + | **New issue URL** | Currently unused. Will be changed in a future release. | + +1. Click **Test settings and save changes**. + +After you configure and enable the Custom Issue Tracker service, you'll see a link on the GitLab +project pages that takes you to that custom issue tracker. ## Referencing issues -- Issues are referenced with `ANYTHING-<ID>`, where `ANYTHING` can be any string in CAPS and `<ID>` -is a number used in the target project of the custom integration (for example, `PROJECT-143`). -- `ANYTHING` is a placeholder to differentiate against GitLab issues, which are referenced with `#<ID>`. You can use a project name or project key to replace it for example. -- When building the hyperlink, the `ANYTHING` part is ignored, and links always point to the address +Issues are referenced with `<ANYTHING>-<ID>` (for example, `PROJECT-143`), where `<ANYTHING>` can be any string in CAPS, and `<ID>` +is a number used in the target project of the custom integration. + +`<ANYTHING>` is a placeholder to differentiate against GitLab issues, which are referenced with `#<ID>`. You can use a project name or project key to replace it for example. + +When building the hyperlink, the `<ANYTHING>` part is ignored, and links always point to the address specified in `issues_url`, so in the example above, `PROJECT-143` would refer to `https://customissuetracker.com/project-name/143`. diff --git a/doc/user/project/integrations/generic_alerts.md b/doc/user/project/integrations/generic_alerts.md index 2234727dd82..57c1e54e48c 100644 --- a/doc/user/project/integrations/generic_alerts.md +++ b/doc/user/project/integrations/generic_alerts.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Generic alerts integration -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13203) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13203) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.8. GitLab can accept alerts from any source via a generic webhook receiver. When you set up the generic alerts integration, a unique endpoint will @@ -18,6 +18,9 @@ create an issue with the payload in the body of the issue. You can always The entire payload will be posted in the issue discussion as a comment authored by the GitLab Alert Bot. +NOTE: **Note** +In GitLab versions 13.1 and greater, you can configure [External Prometheus instances](prometheus.md#external-prometheus-instances) to use this endpoint. + ## Setting up generic alerts To set up the generic alerts integration: @@ -28,7 +31,7 @@ To set up the generic alerts integration: ## Customizing the payload -You can customize the payload by sending the following parameters. All fields are optional: +You can customize the payload by sending the following parameters. All fields other than `title` are optional: | Property | Type | Description | | -------- | ---- | ----------- | @@ -39,6 +42,7 @@ You can customize the payload by sending the following parameters. All fields ar | `monitoring_tool` | String | The name of the associated monitoring tool. | | `hosts` | String or Array | One or more hosts, as to where this incident occurred. | | `severity` | String | The severity of the alert. Must be one of `critical`, `high`, `medium`, `low`, `info`, `unknown`. Default is `critical`. | +| `fingerprint` | String or Array | The unique identifier of the alert. This can be used to group occurrences of the same alert. | TIP: **Payload size:** Ensure your requests are smaller than the [payload application limits](../../../administration/instance_limits.md#generic-alert-json-payloads). @@ -65,5 +69,7 @@ Example payload: "service": "service affected", "monitoring_tool": "value", "hosts": "value", + "severity": "high", + "fingerprint": "d19381d4e8ebca87b55cda6e8eee7385" } ``` diff --git a/doc/user/project/integrations/github.md b/doc/user/project/integrations/github.md index 42d8eadd35e..f0977a4ea76 100644 --- a/doc/user/project/integrations/github.md +++ b/doc/user/project/integrations/github.md @@ -1,6 +1,6 @@ # GitHub project integration **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3836) in GitLab Premium 10.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3836) in GitLab Premium 10.6. GitLab provides an integration for updating the pipeline statuses on GitHub. This is especially useful if using GitLab for CI/CD only. @@ -39,7 +39,7 @@ to configure pipelines to run for open pull requests. #### Static / dynamic status check names > - Introduced in GitLab 11.5: using static status check names as opt-in option. -> - [In GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/issues/9931), static status check names is default behavior for new projects. +> - [In GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/issues/9931), static status check names is default behavior for new projects. This makes it possible to mark these status checks as _Required_ on GitHub. With **Static status check names** enabled on the integration page, your diff --git a/doc/user/project/integrations/gitlab_slack_application.md b/doc/user/project/integrations/gitlab_slack_application.md index 49b6a3f6450..7a827364d41 100644 --- a/doc/user/project/integrations/gitlab_slack_application.md +++ b/doc/user/project/integrations/gitlab_slack_application.md @@ -8,7 +8,7 @@ The GitLab Slack application is only configurable for GitLab.com. It will **not* work for on-premises installations where you can configure the [Slack slash commands](slack_slash_commands.md) service instead. We're planning to make this configurable for all GitLab installations, but there's -no ETA - see [#28164](https://gitlab.com/gitlab-org/gitlab/issues/28164). +no ETA - see [#28164](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). Slack provides a native application which you can enable via your project's integrations on GitLab.com. @@ -32,7 +32,7 @@ integration settings. Keep in mind that you need to have the appropriate permissions for your Slack team in order to be able to install a new application, read more in Slack's -docs on [Adding an app to your team](https://slack.com/help/articles/202035138). +docs on [Adding an app to your workspace](https://slack.com/help/articles/202035138-Add-an-app-to-your-workspace). To enable GitLab's service for your Slack team: diff --git a/doc/user/project/integrations/hangouts_chat.md b/doc/user/project/integrations/hangouts_chat.md index f33833a9c93..f65b31150a9 100644 --- a/doc/user/project/integrations/hangouts_chat.md +++ b/doc/user/project/integrations/hangouts_chat.md @@ -1,6 +1,6 @@ # Hangouts Chat service -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/43756) in GitLab 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/43756) in GitLab 11.2. The Hangouts Chat service sends notifications from GitLab to the room for which the webhook was created. diff --git a/doc/user/project/integrations/img/prometheus_dashboard_yaml_validation_v13_1.png b/doc/user/project/integrations/img/prometheus_dashboard_yaml_validation_v13_1.png Binary files differnew file mode 100644 index 00000000000..08d7d6603d2 --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_dashboard_yaml_validation_v13_1.png diff --git a/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v12_8.png b/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v12_8.png Binary files differdeleted file mode 100644 index 8899852ed04..00000000000 --- a/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v12_8.png +++ /dev/null diff --git a/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v13_1.png b/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v13_1.png Binary files differnew file mode 100644 index 00000000000..56a0a508a1d --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_monitoring_dashboard_v13_1.png diff --git a/doc/user/project/integrations/img/related_links_v13_1.png b/doc/user/project/integrations/img/related_links_v13_1.png Binary files differnew file mode 100644 index 00000000000..4dc141f0e7f --- /dev/null +++ b/doc/user/project/integrations/img/related_links_v13_1.png diff --git a/doc/user/project/integrations/img/slack_configuration.png b/doc/user/project/integrations/img/slack_configuration.png Binary files differdeleted file mode 100644 index 4d5e6ae7856..00000000000 --- a/doc/user/project/integrations/img/slack_configuration.png +++ /dev/null diff --git a/doc/user/project/integrations/jira_cloud_configuration.md b/doc/user/project/integrations/jira_cloud_configuration.md index 9fa92f19e4f..619c94b282b 100644 --- a/doc/user/project/integrations/jira_cloud_configuration.md +++ b/doc/user/project/integrations/jira_cloud_configuration.md @@ -3,7 +3,7 @@ An API token is needed when integrating with Jira Cloud, follow the steps below to create one: -1. Log in to <https://id.atlassian.com/manage/api-tokens> with your email address. +1. Log in to [`id.atlassian.com`](https://id.atlassian.com/manage-profile/security/api-tokens) with your email address. NOTE: **Note** It is important that the user associated with this email address has *write* access diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 6c40e5b9696..0d17ff51372 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -25,7 +25,7 @@ Once enabled, GitLab will automatically detect metrics from known services in th ### Managed Prometheus on Kubernetes -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28916) in GitLab 10.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28916) in GitLab 10.5. GitLab can seamlessly deploy and manage Prometheus on a [connected Kubernetes cluster](../clusters/index.md), making monitoring of your apps easy. @@ -61,7 +61,7 @@ will help you to quickly create a deployment: 1. Navigate to your project's **CI/CD > Pipelines** page, and run a pipeline on any branch. 1. When the pipeline has run successfully, graphs will be available on the **Operations > Metrics** page. - + #### Using the Metrics Dashboard @@ -209,10 +209,19 @@ GitLab supports a limited set of [CI variables](../../../ci/variables/README.md) - `ci_project_namespace` - `ci_project_path` - `ci_environment_name` +- `__range` NOTE: **Note:** Variables for Prometheus queries must be lowercase. +###### __range + +The `__range` variable is useful in Prometheus +[range vector selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors). +Its value is the total number of seconds in the dashboard's time range. +For example, if the dashboard time range is set to 8 hours, the value of +`__range` is `28800s`. + ##### User-defined variables [Variables can be defined](#templating-templating-properties) in a custom dashboard YAML file. @@ -244,7 +253,7 @@ http://gitlab.com/<user>/<project>/-/environments/<environment_id>/metrics?dashb #### Editing additional metrics from the dashboard -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/208976) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208976) in GitLab 12.9. You can edit existing additional custom metrics by clicking the **{ellipsis_v}** **More actions** dropdown and selecting **Edit metric**. @@ -252,7 +261,7 @@ You can edit existing additional custom metrics by clicking the **{ellipsis_v}** ### Defining custom dashboards per project -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/59974) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59974) in GitLab 12.1. By default, all projects include a GitLab-defined Prometheus dashboard, which includes a few key metrics, but you can also define your own custom dashboards. @@ -308,8 +317,8 @@ supported and will not be available in the UI. #### Duplicating a GitLab-defined dashboard -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/37238) in GitLab 12.7. -> - From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37238) in GitLab 12.7. +> - From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. You can save a complete copy of a GitLab defined dashboard along with all custom metrics added to it. Resulting `.yml` file can be customized and adapted to your project. @@ -326,6 +335,42 @@ new branch. If you select your **default** branch, the new dashboard becomes immediately available. If you select another branch, this branch should be merged to your **default** branch first. +#### Dashboard YAML syntax validation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33202) in GitLab 13.1. + +To confirm your dashboard definition contains valid YAML syntax: + +1. Navigate to **{doc-text}** **Repository > Files**. +1. Navigate to your dashboard file in your repository. +1. Review the information pane about the file, displayed above the file contents. + +Files with valid syntax display **Metrics Dashboard YAML definition is valid**, +and files with invalid syntax display **Metrics Dashboard YAML definition is invalid**. + + + +When **Metrics Dashboard YAML definition is invalid** at least one of the following messages is displayed: + +1. `dashboard: can't be blank` [learn more](#dashboard-top-level-properties) +1. `panel_groups: can't be blank` [learn more](#dashboard-top-level-properties) +1. `group: can't be blank` [learn more](#panel-group-panel_groups-properties) +1. `panels: can't be blank` [learn more](#panel-group-panel_groups-properties) +1. `metrics: can't be blank` [learn more](#panel-panels-properties) +1. `title: can't be blank` [learn more](#panel-panels-properties) +1. `query: can't be blank` [learn more](#metrics-metrics-properties) +1. `query_range: can't be blank` [learn more](#metrics-metrics-properties) +1. `unit: can't be blank` [learn more](#metrics-metrics-properties) +1. `YAML syntax: The parsed YAML is too big` + + This is displayed when the YAML file is larger than 1 MB. + +1. `YAML syntax: Invalid configuration format` + + This is displayed when the YAML file is empty or does not contain valid YAML. + +Metrics Dashboard YAML definition validation information is also available as a [GraphQL API field](../../../api/graphql/reference/index.md#metricsdashboard) + #### Dashboard YAML properties Dashboards have several components: @@ -342,16 +387,27 @@ The following tables outline the details of expected properties. | ------ | ------ | ------ | ------ | | `dashboard` | string | yes | Heading for the dashboard. Only one dashboard should be defined per file. | | `panel_groups` | array | yes | The panel groups which should be on the dashboard. | -| `templating` | Hash | no | Top level key under which templating related options can be added. | +| `templating` | hash | no | Top level key under which templating related options can be added. | +| `links` | array | no | Add links to display on the dashboard. | ##### **Templating (`templating`) properties** | Property | Type | Required | Description | | -------- | ---- | -------- | ----------- | -| `variables` | Hash | no | Variables can be defined here. | +| `variables` | hash | yes | Variables can be defined here. | Read the documentation on [templating](#templating-variables-for-metrics-dashboards). +##### **Links (`links`) properties** + +| Property | Type | Required | Description | +| -------- | ---- | -------- | ----------- | +| `url` | string | yes | The address of the link. | +| `title` | string | no | Display title for the link. | +| `type` | string | no | Type of the link. Specifies the link type, can be: `grafana` | + +Read the documentation on [links](#add-related-links-to-custom-dashboards). + ##### **Panel group (`panel_groups`) properties** | Property | Type | Required | Description | @@ -371,6 +427,7 @@ Read the documentation on [templating](#templating-variables-for-metrics-dashboa | `max_value` | number | no | Denominator value used for calculating [percentile based results](#percentile-based-results) | | `weight` | number | no, defaults to order in file | Order to appear within the grouping. Lower number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | | `metrics` | array | yes | The metrics which should be displayed in the panel. Any number of metrics can be displayed when `type` is `area-chart` or `line-chart`, whereas only 3 can be displayed when `type` is `anomaly-chart`. | +| `links` | array | no | Add links to display on the chart's [context menu](#chart-context-menu). | ##### **Axis (`panels[].y_axis`) properties** @@ -472,7 +529,7 @@ Note the following properties:  -Starting in [version 12.8](https://gitlab.com/gitlab-org/gitlab/issues/202696), the y-axis values will automatically scale according to the data. Previously, it always started from 0. +Starting in [version 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/202696), the y-axis values will automatically scale according to the data. Previously, it always started from 0. ##### Anomaly chart @@ -574,7 +631,7 @@ Note the following properties: ##### Stacked column -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30583) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30583) in GitLab 12.8. To add a stacked column panel type to a dashboard, look at the following sample dashboard file: @@ -639,7 +696,7 @@ Note the following properties: ###### Percentile based results -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201946) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201946) in GitLab 12.8. Query results sometimes need to be represented as a percentage value out of 100. You can use the `max_value` property at the root of the panel definition: @@ -662,7 +719,7 @@ For example, if you have a query value of `53.6`, adding `%` as the unit results ##### Heatmaps -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30581) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30581) in GitLab 12.5. To add a heatmap panel type to a dashboard, look at the following sample dashboard file: @@ -699,7 +756,7 @@ Templating variables can be used to make your metrics dashboard more versatile. [dashboard YAML](#dashboard-top-level-properties). Define your variables in the `variables` key, under `templating`. The value of the `variables` key should be a hash, and each key under `variables` -defines a templating variable on the dashboard. +defines a templating variable on the dashboard, and may contain alphanumeric and underscore characters. A variable can be used in a Prometheus query in the same dashboard using the syntax described [here](#using-variables). @@ -766,7 +823,7 @@ templating: ###### Full syntax -This example creates a variable called `variable1`, with a default value of `var1_option_2`. +This example creates a variable called `variable1`, with a default value of `value_option_2`. The label for the text box on the UI will be the value of the `label` key. The dashboard UI will display a dropdown with `Option 1` and `Option 2` as the choices. @@ -789,9 +846,50 @@ templating: default: true # (Optional) This option should be the default value of this variable. ``` +### Add related links to custom dashboards + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216385) in GitLab 13.1. + +You can embed links to other dashboards or external services in your custom +dashboard by adding **Related links** to your dashboard's YAML file. Related links +open in the same tab as the dashboard. Related links can be displayed in the +following locations on your dashboard: + +- At the top of your dashboard as the top level [`links` dashboard property](#dashboard-top-level-properties). +- In charts context menus as the [`links` property of a panel](#panel-panels-properties). + +Related links can contain the following attributes: + +- `url`: The full URL to the link. Required. +- `title`: A phrase describing the link. Optional. If this attribute is not set, + the full URL is used for the link title. +- `type`: A string declaring the type of link. Optional. If set to `grafana`, the + dashboard's time range values are converted to Grafana's time range format and + appended to the `url`. + +The dashboard's time range is appended to the `url` as URL parameters. + +The following example shows two related links (`GitLab.com` and `GitLab Documentation`) +added to a dashboard: + + + +#### Links Syntax + +```yaml +links: + - title: GitLab.com + url: https://gitlab.com + - title: GitLab Documentation + url: https://docs.gitlab.com + - title: Public Grafana playground dashboard + url: https://play.grafana.org/d/000000012/grafana-play-home?orgId=1 + type: grafana +``` + ### View and edit the source file of a custom dashboard -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34779) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5. When viewing a custom dashboard of a project, you can view the original `.yml` file by clicking on the **Edit dashboard** button. @@ -812,7 +910,7 @@ The options are: ### Dashboard Annotations -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/215224) in GitLab 13.0. You can use **Metrics Dashboard Annotations** to mark any important events on @@ -827,6 +925,14 @@ You can create annotations by making requests to the  +#### Retention policy + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211433) in GitLab 13.01. + +To avoid excessive storage space consumption by stale annotations, records attached +to time periods older than two weeks are removed daily. This recurring background +job runs at 1:00 a.m. local server time. + ### Expand panel > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0. @@ -839,7 +945,7 @@ browser, or pressing the <kbd>Esc</kbd> key. ### View Logs **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/122013) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/122013) in GitLab 12.8. If you have [Logs](../clusters/kubernetes_pod_logs.md) enabled, you can navigate from the charts in the dashboard to view Logs by @@ -881,8 +987,8 @@ To remove the alert, click back on the alert icon for the desired metric, and cl #### External Prometheus instances ->- [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9258) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.8. ->- [Moved](https://gitlab.com/gitlab-org/gitlab/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.10. +>- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9258) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.8. +>- [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to [GitLab Core](https://about.gitlab.com/pricing/) in 12.10. For manually configured Prometheus servers, a notify endpoint is provided to use with Prometheus webhooks. If you have manual configuration enabled, an **Alerts** section is added to **Settings > Integrations > Prometheus**. This contains the *URL* and *Authorization Key*. The **Reset Key** button will invalidate the key and generate a new one. @@ -903,12 +1009,15 @@ receivers: In order for GitLab to associate your alerts with an [environment](../../../ci/environments/index.md), you need to configure a `gitlab_environment_name` label on the alerts you set up in Prometheus. The value of this should match the name of your Environment in GitLab. +NOTE: **Note** +In GitLab versions 13.1 and greater, you can configure your manually configured Prometheus server to use the [Generic alerts integration](generic_alerts.md). + ### Taking action on incidents **(ULTIMATE)** ->- [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4925) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. ->- [From GitLab Ultimate 12.5](https://gitlab.com/gitlab-org/gitlab/issues/13401), when GitLab receives a recovery alert, it will automatically close the associated issue. +>- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4925) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.11. +>- [From GitLab Ultimate 12.5](https://gitlab.com/gitlab-org/gitlab/-/issues/13401), when GitLab receives a recovery alert, it will automatically close the associated issue. -Alerts can be used to trigger actions, like opening an issue automatically (enabled by default since `12.1`). To configure the actions: +Alerts can be used to trigger actions, like opening an issue automatically (disabled by default since `13.1`). To configure the actions: 1. Navigate to your project's **Settings > Operations > Incidents**. 1. Enable the option to create issues. @@ -931,14 +1040,14 @@ When GitLab receives a **Recovery Alert**, it will automatically close the assoc To further customize the issue, you can add labels, mentions, or any other supported [quick action](../quick_actions.md) in the selected issue template, which will apply to all incidents. To limit quick actions or other information to only specific types of alerts, use the `annotations/gitlab_incident_markdown` field. -Since [version 12.2](https://gitlab.com/gitlab-org/gitlab-foss/issues/63373), GitLab will tag each incident issue with the `incident` label automatically. If the label does not yet exist, it will be created automatically as well. +Since [version 12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/63373), GitLab will tag each incident issue with the `incident` label automatically. If the label does not yet exist, it will be created automatically as well. If the metric exceeds the threshold of the alert for over 5 minutes, an email will be sent to all [Maintainers and Owners](../../permissions.md#project-members-permissions) of the project. ## Determining the performance impact of a merge > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10408) in GitLab 9.2. -> - GitLab 9.3 added the [numeric comparison](https://gitlab.com/gitlab-org/gitlab-foss/issues/27439) of the 30 minute averages. +> - GitLab 9.3 added the [numeric comparison](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/27439) of the 30 minute averages. Developers can view the performance impact of their changes within the merge request workflow. @@ -1031,7 +1140,7 @@ For [manually configured Prometheus instances](#manual-configuration-of-promethe ### Embedding Cluster Health Charts **(ULTIMATE)** -> [Introduced](<https://gitlab.com/gitlab-org/gitlab/issues/40997>) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. +> [Introduced](<https://gitlab.com/gitlab-org/gitlab/-/issues/40997>) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.9. [Cluster Health Metrics](../clusters/index.md#monitoring-your-kubernetes-cluster-ultimate) can also be embedded in [GitLab-flavored Markdown](../../markdown.md). @@ -1075,7 +1184,7 @@ This will render like so: #### Embedding charts via integration with Grafana HTTP API -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31376) in GitLab 12.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31376) in GitLab 12.5. Each project can support integration with one Grafana instance. This configuration allows a user to copy a link to a panel in Grafana, then paste it into a GitLab Markdown field. The chart will be rendered in the GitLab chart format. diff --git a/doc/user/project/integrations/prometheus_units.md b/doc/user/project/integrations/prometheus_units.md index 691d20e5de2..7b216ced1cc 100644 --- a/doc/user/project/integrations/prometheus_units.md +++ b/doc/user/project/integrations/prometheus_units.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Unit formats reference -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201999) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201999) in GitLab 12.9. You can select units to format your charts by adding `format` to your [axis configuration](prometheus.md#dashboard-yaml-properties). @@ -19,7 +19,7 @@ Currently, your [internationalization and localization options](https://en.wikip For generic or default data, numbers are formatted according to the current locale in [engineering notation](https://en.wikipedia.org/wiki/Engineering_notation). -While an [engineering notation exists for the web](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NumberFormat), GitLab uses a version based off the [scientific notation](https://en.wikipedia.org/wiki/Scientific_notation). GitLab formatting acts in accordance with SI prefixes. For example, using GitLab notation, `1500.00` becomes `1.5k` instead of `1.5E3`. Keep this distinction in mind when using the engineering notation for your metrics. +While an [engineering notation exists for the web](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat), GitLab uses a version based off the [scientific notation](https://en.wikipedia.org/wiki/Scientific_notation). GitLab formatting acts in accordance with SI prefixes. For example, using GitLab notation, `1500.00` becomes `1.5k` instead of `1.5E3`. Keep this distinction in mind when using the engineering notation for your metrics. Formats: `engineering` diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index 419340c14ef..79fc8eceddf 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -23,7 +23,23 @@ The Slack Notifications Service allows your GitLab project to send events (e.g. Your Slack team will now start receiving GitLab event notifications as configured. - +### Triggers available for Slack notifications + +The following triggers are available for Slack notifications: + +- **Push**: Triggered by a push to the repository. +- **Issue**: Triggered when an issue is created, updated, or closed. +- **Confidential issue**: Triggered when a confidential issue is created, + updated, or closed. +- **Merge request**: Triggered when a merge request is created, updated, or + merged. +- **Note**: Triggered when someone adds a comment. +- **Confidential note**: Triggered when someone adds a confidential note. +- **Tag push**: Triggered when a new tag is pushed to the repository. +- **Pipeline**: Triggered when a pipeline status changes. +- **Wiki page**: Triggered when a wiki page is created or updated. +- **Deployment**: Triggered when a deployment finishes. +- **Alert**: Triggered when a new, unique alert is recorded. ## Troubleshooting diff --git a/doc/user/project/integrations/webex_teams.md b/doc/user/project/integrations/webex_teams.md index a6e688887b6..10735e33746 100644 --- a/doc/user/project/integrations/webex_teams.md +++ b/doc/user/project/integrations/webex_teams.md @@ -4,7 +4,7 @@ You can configure GitLab to send notifications to a Webex Teams space. ## Create a webhook for the space -1. Go to the [Incoming Webooks app page](https://apphub.webex.com/teams/applications/incoming-webhooks-cisco-systems). +1. Go to the [Incoming Webhooks app page](https://apphub.webex.com/teams/applications/incoming-webhooks-cisco-systems). 1. Click **Connect** and log in to Webex Teams, if required. 1. Enter a name for the webhook and select the space that will receive the notifications. 1. Click **ADD**. diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 89e84dda0e8..5a0ca03a646 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -89,7 +89,7 @@ You can turn this off in the webhook settings in your GitLab projects. ## Branch filtering -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/20338) in GitLab 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/20338) in GitLab 11.3. Push events can be filtered by branch using a branch name or wildcard pattern to limit which push events are sent to your webhook endpoint. By default the diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 9903a711987..89b17609698 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Issue Boards > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5554) in [GitLab 8.11](https://about.gitlab.com/releases/2016/08/22/gitlab-8-11-released/#issue-board). @@ -190,7 +196,7 @@ as shown in the following table: ### Multiple issue boards > - [Introduced](https://about.gitlab.com/releases/2016/10/22/gitlab-8-13-released/) in GitLab 8.13. -> - Multiple issue boards per project [moved](https://gitlab.com/gitlab-org/gitlab-foss/issues/53811) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 12.1. +> - Multiple issue boards per project [moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53811) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 12.1. > - Multiple issue boards per group are available in [GitLab Premium](https://about.gitlab.com/pricing/). Multiple issue boards allow for more than one issue board for a given project or group. @@ -290,7 +296,7 @@ Multiple group issue boards were originally [introduced](https://about.gitlab.co ### Assignee lists **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5784) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5784) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. Like in a regular list that shows all issues with a chosen label, you can add an assignee list that shows all issues assigned to a user. @@ -311,7 +317,7 @@ To remove an assignee list, just as with a label list, click the trash icon. ### Milestone lists **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/6469) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6469) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. You're also able to create lists of a milestone. These are lists that filter issues by the assigned milestone, giving you more freedom and visibility on the issue board. To add a milestone list: @@ -330,7 +336,7 @@ As in other list types, click the trash icon to remove a list. ## Work In Progress limits **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11403) in GitLab 12.7 +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11403) in GitLab 12.7 You can set Work In Progress (WIP) limits per issues list. When a limit is set, the list's header shows the number of issues in the list and the soft limit of issues. @@ -352,7 +358,7 @@ To set a WIP limit for a list: ## Blocked issues -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34723) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34723) in GitLab 12.8. If an issue is blocked by another issue, an icon appears next to its title to indicate its blocked status. @@ -370,7 +376,7 @@ status. - [Filter issues](#filter-issues) that appear across your issue board. - [Create workflows](#create-workflows). - [Drag issues between lists](#drag-issues-between-lists). -- [Mulit-select issue cards](#multi-select-issue-cards). +- [Multi-select issue cards](#multi-select-issue-cards). - [Re-order issues in lists](#issue-ordering-in-a-list). - Drag and reorder the lists. - Change issue labels (by dragging an issue between lists). @@ -501,7 +507,7 @@ When dragging issues between lists, different behavior occurs depending on the s ### Multi-select issue cards -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/18954) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18954) in GitLab 12.4. You can select multiple issue cards, then drag the group to another position within the list, or to another list. This makes it faster to reorder many issues at once. diff --git a/doc/user/project/issues/associate_zoom_meeting.md b/doc/user/project/issues/associate_zoom_meeting.md index 3bfd24c9654..a026854a947 100644 --- a/doc/user/project/issues/associate_zoom_meeting.md +++ b/doc/user/project/issues/associate_zoom_meeting.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Associate a Zoom meeting with an issue > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609) in GitLab 12.4. diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md index a292ce64af9..602a6d79848 100644 --- a/doc/user/project/issues/confidential_issues.md +++ b/doc/user/project/issues/confidential_issues.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Confidential issues > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3282) in GitLab 8.6. @@ -80,7 +86,7 @@ project's search results respectively. ## Merge Requests for Confidential Issues -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/58583) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58583) in GitLab 12.1. To help prevent confidential information being leaked from a public project in the process of resolving a confidential issue, confidential issues can be diff --git a/doc/user/project/issues/crosslinking_issues.md b/doc/user/project/issues/crosslinking_issues.md index e774fd4480b..6cc31a45309 100644 --- a/doc/user/project/issues/crosslinking_issues.md +++ b/doc/user/project/issues/crosslinking_issues.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Crosslinking Issues Please read through the [GitLab Issue Documentation](index.md) for an overview on GitLab Issues. diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index 64f56221632..981c2a7c34a 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -3,10 +3,6 @@ > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/660) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212566) to GitLab Core in 13.0. -CAUTION: **Warning:** -This an **alpha** feature and is subject to change at any time without -prior notice. - ## Overview Design Management allows you to upload design assets (wireframes, mockups, etc.) @@ -45,19 +41,19 @@ If the requirements are not met, the **Designs** tab displays a message to the u Files uploaded must have a file extension of either `png`, `jpg`, `jpeg`, `gif`, `bmp`, `tiff` or `ico`. -Support for [SVG files](https://gitlab.com/gitlab-org/gitlab/issues/12771) -and [PDFs](https://gitlab.com/gitlab-org/gitlab/issues/32811) is planned for a future release. +Support for [SVG files](https://gitlab.com/gitlab-org/gitlab/-/issues/12771) +and [PDFs](https://gitlab.com/gitlab-org/gitlab/-/issues/32811) is planned for a future release. ## Limitations - Design uploads are limited to 10 files at a time. - Design Management data - [isn't deleted when a project is destroyed](https://gitlab.com/gitlab-org/gitlab/issues/13429) yet. -- Design Management data [won't be moved](https://gitlab.com/gitlab-org/gitlab/issues/13426) - when an issue is moved, nor [deleted](https://gitlab.com/gitlab-org/gitlab/issues/13427) + [isn't deleted when a project is destroyed](https://gitlab.com/gitlab-org/gitlab/-/issues/13429) yet. +- Design Management data [won't be moved](https://gitlab.com/gitlab-org/gitlab/-/issues/13426) + when an issue is moved, nor [deleted](https://gitlab.com/gitlab-org/gitlab/-/issues/13427) when an issue is deleted. - From GitLab 12.7, Design Management data [can be replicated](../../../administration/geo/replication/datatypes.md#limitations-on-replicationverification) - by Geo but [not verified](https://gitlab.com/gitlab-org/gitlab/issues/32467). + by Geo but [not verified](https://gitlab.com/gitlab-org/gitlab/-/issues/32467). - Only the latest version of the designs can be deleted. - Deleted designs cannot be recovered but you can see them on previous designs versions. @@ -71,8 +67,8 @@ Navigate to the **Design Management** page from any issue by clicking the **Desi To upload design images, click the **Upload Designs** button and select images to upload. -[Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9, -you can drag and drop designs onto the dedicated dropzone to upload them. +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9, +you can drag and drop designs onto the dedicated drop zone to upload them.  @@ -91,7 +87,7 @@ Copy-and-pasting has some limitations: - Copy/pasting designs is not supported on Internet Explorer. Designs with the same filename as an existing uploaded design will create a new version -of the design, and will replace the previous version. [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9, dropping a design on an existing uploaded design will also create a new version, +of the design, and will replace the previous version. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9, dropping a design on an existing uploaded design will also create a new version, provided the filenames are the same. Designs cannot be added if the issue has been moved, or its @@ -123,19 +119,19 @@ to help summarize changes between versions. ### Exploring designs by zooming -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13217) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13217) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. Designs can be explored in greater detail by zooming in and out of the image. Control the amount of zoom with the `+` and `-` buttons at the bottom of the image. While zoomed, you can still [start new discussions](#starting-discussions-on-designs) on the image, and see any existing ones. -[Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197324) in GitLab 12.10, while zoomed in, +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/197324) in GitLab 12.10, while zoomed in, you can click-and-drag on the image to move around it.  ## Deleting designs -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/11089) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11089) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. There are two ways to delete designs: manually delete them individually, or select a few of them to delete at once, @@ -169,7 +165,7 @@ A pin is added to the image, identifying the discussion's location.  -[Introduced](https://gitlab.com/gitlab-org/gitlab/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8, +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34353) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8, you can adjust a pin's position by dragging it around the image. This is useful for when your design layout has changed between revisions, or if you need to move an existing pin to add a new one in its place. @@ -180,3 +176,60 @@ Different discussions have different pin numbers: From GitLab 12.5 on, new discussions will be outputted to the issue activity, so that everyone involved can participate in the discussion. + +## Resolve Design threads + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13049) in GitLab 13.1. + +Discussion threads can be resolved on Designs. You can mark a thread as resolved +or unresolved by clicking the **Resolve thread** icon at the first comment of the +discussion. + + + +Pinned comments can also be resolved or unresolved in their threads. +When replying to a comment, you will see a checkbox that you can click in order to resolve or unresolve +the thread once published. + + + +## Referring to designs in Markdown + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217160) in **GitLab 13.1**. +> - It is deployed behind a feature flag, disabled by default. +> - It is disabled on GitLab.com. +> - It is not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-design-references-core-only). **(CORE ONLY)** + +We support referring to designs in [Markdown](../../markdown.md), which is available +throughout the application, including in merge request and issue descriptions, in discussions and comments, and in wiki pages. + +At present, full URL references are supported. For example, if we refer to a design +somewhere with: + +```markdown +See https://gitlab.com/your-group/your-project/-/issues/123/designs/homescreen.png +``` + +This will be rendered as: + +> See [#123[homescreen.png]](https://gitlab.com/your-group/your-project/-/issues/123/designs/homescreen.png) + +### Enable or disable design references **(CORE ONLY)** + +Design reference parsing is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:design_management_reference_filter_gfm_pipeline) +``` + +To disable it: + +```ruby +Feature.disable(:design_management_reference_filter_gfm_pipeline) +``` diff --git a/doc/user/project/issues/due_dates.md b/doc/user/project/issues/due_dates.md index 0be0cdd11bd..56fb4ca5cc7 100644 --- a/doc/user/project/issues/due_dates.md +++ b/doc/user/project/issues/due_dates.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Due dates > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3614) in GitLab 8.7. diff --git a/doc/user/project/issues/img/new_issue.png b/doc/user/project/issues/img/new_issue.png Binary files differdeleted file mode 100644 index 07d65a93070..00000000000 --- a/doc/user/project/issues/img/new_issue.png +++ /dev/null diff --git a/doc/user/project/issues/img/new_issue_v13_1.png b/doc/user/project/issues/img/new_issue_v13_1.png Binary files differnew file mode 100644 index 00000000000..a66846c234e --- /dev/null +++ b/doc/user/project/issues/img/new_issue_v13_1.png diff --git a/doc/user/project/issues/img/resolve_design-discussion_checkbox_v13_1.png b/doc/user/project/issues/img/resolve_design-discussion_checkbox_v13_1.png Binary files differnew file mode 100644 index 00000000000..bd9d1f7a77c --- /dev/null +++ b/doc/user/project/issues/img/resolve_design-discussion_checkbox_v13_1.png diff --git a/doc/user/project/issues/img/resolve_design-discussion_icon_v13_1.png b/doc/user/project/issues/img/resolve_design-discussion_icon_v13_1.png Binary files differnew file mode 100644 index 00000000000..5cd005f0799 --- /dev/null +++ b/doc/user/project/issues/img/resolve_design-discussion_icon_v13_1.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 22221531360..06a80672769 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Issues Issues are the fundamental medium for collaborating on ideas and planning work in GitLab. @@ -151,7 +157,7 @@ context, such as past work, dependencies, or duplicates. ### Crosslinking issues -You can [crosslink issues](crosslinking_issues.md) by referencing an issue from another +You can [cross-link issues](crosslinking_issues.md) by referencing an issue from another issue or merge request by including its URL or ID. The referenced issue displays a message in the Activity stream about the reference, with a link to the other issue or MR. @@ -173,7 +179,7 @@ requires [GraphQL](../../../api/graphql/index.md) to be enabled. ### Health status **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36427) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36427) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. To help you track the status of your issues, you can assign a status to each issue to flag work that's progressing as planned or needs attention to keep on schedule: diff --git a/doc/user/project/issues/issue_data_and_actions.md b/doc/user/project/issues/issue_data_and_actions.md index 65e21566d5d..6d34f91d37f 100644 --- a/doc/user/project/issues/issue_data_and_actions.md +++ b/doc/user/project/issues/issue_data_and_actions.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Issue Data and Actions Please read through the [GitLab Issue Documentation](index.md) for an overview on GitLab Issues. @@ -185,7 +191,7 @@ The plain text title and description of the issue fill the top center of the iss The description fully supports [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-gfm), allowing many formatting options. -> [Since GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/10103), changes to an issue's description are listed in the [issue history](#issue-history).**(STARTER)** +> [Since GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/10103), changes to an issue's description are listed in the [issue history](#issue-history).**(STARTER)** ### Mentions @@ -296,7 +302,7 @@ You can also close the issue from here, so you don't need to scroll to the top o ### Zoom meetings -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/31103) in GitLab 12.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31103) in GitLab 12.3. You can attach and remove Zoom meetings to issues using the `/zoom` and `/remove_zoom` [quick actions](../quick_actions.md) as part of [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-gfm). @@ -305,3 +311,9 @@ Attaching a [Zoom](https://zoom.us) call an issue results in a **Join Zoom meeting** button at the top of the issue, just under the header. Read more how to [add or remove a zoom meeting](associate_zoom_meeting.md). + +### Publish an issue **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. + +If a status page application is associated with the project, you can use the `/publish` [quick action](../quick_actions.md) to publish the issue. Refer to [GitLab Status Page](../status_page/index.md) for more information. diff --git a/doc/user/project/issues/issue_weight.md b/doc/user/project/issues/issue_weight.md index 74f607dd407..23c1520294d 100644 --- a/doc/user/project/issues/issue_weight.md +++ b/doc/user/project/issues/issue_weight.md @@ -1,5 +1,8 @@ --- disqus_identifier: 'https://docs.gitlab.com/ee/workflow/issue_weight.html' +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Issue weight **(STARTER)** diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 4e329889e7c..08e3164b2eb 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -1,21 +1,28 @@ -# Managing Issues +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Managing issues [GitLab Issues](index.md) are the fundamental medium for collaborating on ideas and planning work in GitLab. [Creating](#create-a-new-issue), [moving](#moving-issues), [closing](#closing-issues), and [deleting](#deleting-issues) are key actions that you can do with issues. -## Create a new Issue +## Create a new issue -When you create a new issue, you'll be prompted to fill in the [data and fields of the issue](issue_data_and_actions.md), as illustrated below. If you know -the values you want to assign to an issue, you can use the [Quick actions](../quick_actions.md) -feature to input values, instead of selecting them from lists. +When you create a new issue, you'll be prompted to fill in the [data and fields of the issue](issue_data_and_actions.md), +as illustrated below. If you know the values you want to assign to an issue, you can use the +[Quick actions](../quick_actions.md) feature to input values, instead of selecting them from lists. - +While creating an issue, you can associate it to an existing epic from current group by +selecting it using **Epic** dropdown. -### Accessing the new Issue form +### Accessing the New Issue form -There are many ways to get to the new Issue form from within a project: +There are many ways to get to the New Issue form from within a project: - Navigate to your **Project's Dashboard** > **Issues** > **New Issue**: @@ -36,9 +43,28 @@ There are many ways to get to the new Issue form from within a project:  +### Elements of the New Issue form + +> Ability to add the new issue to an epic [was introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13847) +> in [GitLab Premium](https://about.gitlab.com/pricing/) 13.1. + + + +When you're creating a new issue, these are the fields you can fill in: + +- Title +- Description +- Checkbox to make the issue confidential +- Assignee +- Weight +- Epic **(PREMIUM)** +- Due date +- Milestone +- Labels + ### New issue from the group-level Issue Tracker -Go to the Group dashboard and click "Issues" in the sidebar to visit the Issue Tracker +Go to the Group dashboard and click **Issues** in the sidebar to visit the Issue Tracker for all projects in your Group. Select the project you'd like to add an issue for using the dropdown button at the top-right of the page. @@ -107,11 +133,11 @@ field). Follow these examples to form your new issue URL with prefilled fields. - For a new issue in the GitLab Community Edition project with a pre-filled title - and a pre-filled description, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea` + and a pre-filled description, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/-/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea` - For a new issue in the GitLab Community Edition project with a pre-filled title - and a pre-filled description template, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/issues/new?issue[title]=Validate%20new%20concept&issuable_template=Research%20proposal` + and a pre-filled description template, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/-/issues/new?issue[title]=Validate%20new%20concept&issuable_template=Research%20proposal` - For a new issue in the GitLab Community Edition project with a pre-filled title, - a pre-filled description, and the confidential flag set, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea&issue[confidential]=true` + a pre-filled description, and the confidential flag set, the URL would be `https://gitlab.com/gitlab-org/gitlab-foss/-/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea&issue[confidential]=true` ## Moving Issues @@ -147,7 +173,7 @@ issues.each do |issue| end; nil ``` -## Closing Issues +## Closing issues When you decide that an issue is resolved, or no longer needed, you can close the issue using the close button: @@ -226,6 +252,8 @@ when used from the command line with `git commit -m`. #### Disabling automatic issue closing +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/19754) in GitLab 12.7. + The automatic issue closing feature can be disabled on a per-project basis within the [project's repository settings](../settings/index.md). Referenced issues will still be displayed as such but won't be closed automatically. @@ -243,7 +271,7 @@ In order to change the default issue closing pattern, GitLab administrators must [`gitlab.rb` or `gitlab.yml` file](../../../administration/issue_closing_pattern.md) of your installation. -## Deleting Issues +## Deleting issues > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2982) in GitLab 8.6 diff --git a/doc/user/project/issues/multiple_assignees_for_issues.md b/doc/user/project/issues/multiple_assignees_for_issues.md index dff6a6031d7..c11977f5bee 100644 --- a/doc/user/project/issues/multiple_assignees_for_issues.md +++ b/doc/user/project/issues/multiple_assignees_for_issues.md @@ -1,6 +1,12 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Multiple Assignees for Issues **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1904) in [GitLab Starter 9.2](https://about.gitlab.com/releases/2017/05/22/gitlab-9-2-released/#multiple-assignees-for-issues). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1904) in [GitLab Starter 9.2](https://about.gitlab.com/releases/2017/05/22/gitlab-9-2-released/#multiple-assignees-for-issues). ## Overview diff --git a/doc/user/project/issues/related_issues.md b/doc/user/project/issues/related_issues.md index 8002b528a80..445c28492df 100644 --- a/doc/user/project/issues/related_issues.md +++ b/doc/user/project/issues/related_issues.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Related issues **(STARTER)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1797) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.4. diff --git a/doc/user/project/issues/sorting_issue_lists.md b/doc/user/project/issues/sorting_issue_lists.md index d52b629bfed..7cbd9906800 100644 --- a/doc/user/project/issues/sorting_issue_lists.md +++ b/doc/user/project/issues/sorting_issue_lists.md @@ -1,4 +1,10 @@ -# Sorting and Ordering Issue Lists +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Sorting and ordering issue lists You can sort a list of issues several ways, including by issue creation date, milestone due date, etc. The available sorting options can change based on the context of the list. @@ -9,7 +15,7 @@ similar to [issue boards](../issue_board.md#issue-ordering-in-a-list). ## Manual sorting -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/62178) in GitLab 12.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62178) in GitLab 12.2. When you select **Manual** sorting, you can change the order by dragging and dropping the issues. The changed order will persist. Everyone who visits the same list will see the reordered list, with some exceptions. diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index f3b59147d5b..406938519b1 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -1,3 +1,9 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Labels ## Overview @@ -131,7 +137,7 @@ to the project: ## Scoped labels **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9175) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9175) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. Scoped labels allow teams to use the label feature to annotate issues, merge requests and epics with mutually exclusive labels. This can enable more complicated workflows @@ -203,8 +209,8 @@ to label notifications for the project only, or the whole group. ## Label priority -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/14189) in GitLab 8.9. -> - Priority sorting is based on the highest priority label only. [This discussion](https://gitlab.com/gitlab-org/gitlab/issues/14523) considers changing this. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14189) in GitLab 8.9. +> - Priority sorting is based on the highest priority label only. [This discussion](https://gitlab.com/gitlab-org/gitlab/-/issues/14523) considers changing this. Labels can have relative priorities, which are used in the **Label priority** and **Priority** sort orders of the epic, issue, and merge request list pages. Prioritization @@ -229,7 +235,7 @@ If you sort by `Label priority`, GitLab uses this sort comparison order: 1. Items without a prioritized label. Ties are broken arbitrarily. Note that only the highest prioritized label is checked, -and labels with a lower priority are ignored. See this [related issue](https://gitlab.com/gitlab-org/gitlab/issues/14523) +and labels with a lower priority are ignored. See this [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/14523) for more information.  diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index 9020dc335b6..787a74b0065 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -8,7 +8,7 @@ You should have Maintainer or Owner [permissions](../../permissions.md) to add or import a new user to your project. To view, edit, add, and remove project's members, go to your -project's **Settings > Members**. +project's **Members**. ## Inherited membership @@ -27,7 +27,7 @@ From the image above, we can deduce the following things: - Administrator is the Owner and member of **all** groups and for that reason, there is an indication of an ancestor group and inherited Owner permissions. -[From GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/21727), you can filter this list +[From GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/21727), you can filter this list using the dropdown on the right side:  diff --git a/doc/user/project/members/share_project_with_groups.md b/doc/user/project/members/share_project_with_groups.md index d0ceac3e1f3..033d69cbbfa 100644 --- a/doc/user/project/members/share_project_with_groups.md +++ b/doc/user/project/members/share_project_with_groups.md @@ -18,7 +18,7 @@ This is where the group sharing feature can be of use. To share 'Project Acme' with the 'Engineering' group: -1. For 'Project Acme' use the left navigation menu to go to **Settings > Members** +1. For 'Project Acme' use the left navigation menu to go to **Members**  diff --git a/doc/user/project/merge_requests/accessibility_testing.md b/doc/user/project/merge_requests/accessibility_testing.md index 427761281f6..417b85a6082 100644 --- a/doc/user/project/merge_requests/accessibility_testing.md +++ b/doc/user/project/merge_requests/accessibility_testing.md @@ -25,16 +25,6 @@ Accessibility Report in the merge request widget area:  -This widget comes with the `:accessibility_report_view` feature flag disabled by default while we test feature stability. -Once we have determined the widget is stable, this feature will be enabled by default. - -To enable this feature, ask a GitLab administrator with [Rails console access](../../../administration/feature_flags.md#how-to-enable-and-disable-features-behind-flags) to run the -following command: - -```ruby -Feature.enable(:accessibility_report_view) -``` - ## Configure Accessibility Testing This example shows how to run [pa11y](https://pa11y.org/) diff --git a/doc/user/project/merge_requests/browser_performance_testing.md b/doc/user/project/merge_requests/browser_performance_testing.md index 0fa3d7be265..390d480724d 100644 --- a/doc/user/project/merge_requests/browser_performance_testing.md +++ b/doc/user/project/merge_requests/browser_performance_testing.md @@ -60,7 +60,7 @@ on your code by using GitLab CI/CD and [sitespeed.io](https://www.sitespeed.io) using Docker-in-Docker. 1. First, set up GitLab Runner with a - [docker-in-docker build](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). + [Docker-in-Docker build](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). 1. After configuring the Runner, add a new job to `.gitlab-ci.yml` that generates the expected report. 1. Define the `performance` job according to your version of GitLab: @@ -125,7 +125,7 @@ Key metrics are automatically extracted and shown in the merge request widget. ### Configuring degradation threshold -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27599) in GitLab 13.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27599) in GitLab 13.0. You can configure the sensitivity of degradation alerts to avoid getting alerts for minor drops in metrics. This is done by setting the `DEGRADATION_THRESHOLD` variable. In the example below, the alert will only show up diff --git a/doc/user/project/merge_requests/cherry_pick_changes.md b/doc/user/project/merge_requests/cherry_pick_changes.md index 3e76b9ec6b9..6685c50c1ed 100644 --- a/doc/user/project/merge_requests/cherry_pick_changes.md +++ b/doc/user/project/merge_requests/cherry_pick_changes.md @@ -33,7 +33,7 @@ request thread crosslinking the new commit and the existing merge request. Each deployment's [list of associated merge requests](../../../api/deployments.md#list-of-merge-requests-associated-with-a-deployment) will include cherry-picked merge commits. NOTE: **Note:** -We only track cherry-pick executed from GitLab (both UI and API). Support for [tracking cherry-picked commits through the command line](https://gitlab.com/gitlab-org/gitlab/issues/202215) is planned for a future release. +We only track cherry-pick executed from GitLab (both UI and API). Support for [tracking cherry-picked commits through the command line](https://gitlab.com/gitlab-org/gitlab/-/issues/202215) is planned for a future release. ## Cherry-picking a commit diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index beb90e30906..7b4d4b668d5 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -12,7 +12,7 @@ source code quality using GitLab Code Quality. Code Quality: - Uses [Code Climate Engines](https://codeclimate.com), which are - free and open source. Code Quality doesn't require a Code Climate + free and open source. Code Quality does not require a Code Climate subscription. - Runs in [pipelines](../../../ci/pipelines/index.md) using a Docker image built in the [GitLab Code @@ -67,10 +67,10 @@ This example shows how to run Code Quality on your code by using GitLab CI/CD an First, you need GitLab Runner configured: -- For the [docker-in-docker workflow](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). +- For the [Docker-in-Docker workflow](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). - With enough disk space to handle generated Code Quality files. For example on the [GitLab project](https://gitlab.com/gitlab-org/gitlab) the files are approximately 7 GB. -Once you set up the Runner, include the CodeQuality template in your CI config: +Once you set up the Runner, include the Code Quality template in your CI configuration: ```yaml include: @@ -80,10 +80,9 @@ include: The above example will create a `code_quality` job in your CI/CD pipeline which will scan your source code for code quality issues. The report will be saved as a [Code Quality report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality-starter) -that you can later download and analyze. Due to implementation limitations we always -take the latest Code Quality artifact available. +that you can later download and analyze. -It is also possible to override the URL to the Code Quality image by +It's also possible to override the URL to the Code Quality image by setting the `CODE_QUALITY_IMAGE` variable. This is particularly useful if you want to lock in a specific version of Code Quality, or use a fork of it: @@ -108,7 +107,7 @@ code_quality: paths: [gl-code-quality-report.json] ``` -The included `code_quality` job is running in the `test` stage, so it needs to be included in your CI config, like so: +The included `code_quality` job is running in the `test` stage, so it needs to be included in your CI configuration, like so: ```yaml stages: @@ -120,7 +119,7 @@ This information will be automatically extracted and shown right in the merge re CAUTION: **Caution:** On self-managed instances, if a malicious actor compromises the Code Quality job -definition they will be able to execute privileged docker commands on the Runner +definition they will be able to execute privileged Docker commands on the Runner host. Having proper access control policies mitigates this attack vector by allowing access only to trusted actors. @@ -129,9 +128,8 @@ allowing access only to trusted actors. CAUTION: **Caution:** Before GitLab 11.5, Code Quality job and artifact had to be named specifically to automatically extract report data and show it in the merge request widget. While these -old job definitions are still maintained they have been deprecated and may be removed -in the next major release, GitLab 12.0. You are advised to update your current `.gitlab-ci.yml` -configuration to reflect that change. +old job definitions are still maintained they have been deprecated and are no longer supported on GitLab 12.0 or higher. +You're advised to update your `.gitlab-ci.yml` configuration to reflect that change. For GitLab 11.5 and later, the job should look like: @@ -157,7 +155,7 @@ code_quality: In GitLab 12.6, Code Quality switched to the [new versioning scheme](https://gitlab.com/gitlab-org/ci-cd/codequality#versioning-and-release-cycle). -It is highly recommended to include the Code Quality template as shown in the +It's highly recommended to include the Code Quality template as shown in the [example configuration](#example-configuration), which uses the new versioning scheme. If not using the template, the `SP_VERSION` variable can be hardcoded to use the new image versions: @@ -241,7 +239,7 @@ do this: [Code Quality report artifact](../../../ci/pipelines/job_artifacts.md#artifactsreportscodequality-starter). 1. Configure your tool to generate the Code Quality report artifact as a JSON - file that implements subset of the [Code Climate + file that implements a subset of the [Code Climate spec](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types). The Code Quality report artifact JSON file must contain an array of objects @@ -288,28 +286,28 @@ Once the Code Quality job has completed: [downloadable artifact](../../../ci/pipelines/job_artifacts.md#downloading-artifacts) for the `code_quality` job. -If multiple jobs in a pipeline generate a code quality artifact, only the artifact from -the last created job (the job with the largest job ID) is used. To avoid confusion, -configure only one job to generate a code quality artifact. +## Troubleshooting -If the Code Quality report doesn't have anything to compare to, no information -will be displayed in the merge request area. That is the case when you add the -Code Quality job in your `.gitlab-ci.yml` for the very first time. -Consecutive merge requests will have something to compare to and the Code Quality -report will be shown properly. +### No Code Quality report is displayed in a Merge Request -These reports will only be available as long as the Code Quality artifact(s) required to generate -them are also available. See -[`artifacts:expire_in`](../../../ci/yaml/README.md#artifactsexpire_in) for more details. +This can be due to multiple reasons: -<!-- ## Troubleshooting +- You just added the Code Quality job in your `.gitlab-ci.yml`. The report does not + have anything to compare to yet, so no information can be displayed. Future merge + requests will have something to compare to. +- If no [degradation or error is detected](https://docs.codeclimate.com/docs/maintainability#section-checks), + nothing will be displayed. +- The [`artifacts:expire_in`](../../../ci/yaml/README.md#artifactsexpire_in) CI/CD + setting can cause the Code Quality artifact(s) to expire faster than desired. +- Large `codeclimate.json` files (esp. >10 MB) are [known to prevent the report from being displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/2737). + As a work-around, try removing [properties](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types) + that are [ignored by GitLab](#implementing-a-custom-tool). You can: + - Configure the Code Quality tool to not output those types. + - Use `sed`, `awk` or similar commands in the `.gitlab-ci.yml` script to + edit the `codeclimate.json` before the job completes. -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +### Only a single Code Quality report is displayed, but more are defined -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +GitLab only uses the Code Quality artifact from the latest created job (with the largest job ID). +If multiple jobs in a pipeline generate a code quality artifact, those of earlier jobs are ignored. +To avoid confusion, configure only one job to generate a `codeclimate.json`. diff --git a/doc/user/project/merge_requests/creating_merge_requests.md b/doc/user/project/merge_requests/creating_merge_requests.md index 544727380e7..50d5decc2cc 100644 --- a/doc/user/project/merge_requests/creating_merge_requests.md +++ b/doc/user/project/merge_requests/creating_merge_requests.md @@ -12,8 +12,8 @@ to familiarize yourself with the concept, the terminology, and to learn what you can do with them. Every merge request starts by creating a branch. You can either -do it locally through the command line, via a Git CLI application, -or through the GitLab UI. +do it locally through the [command line](#new-merge-request-from-your-local-environment), via a Git CLI application, +or through the [GitLab UI](#new-merge-request-from-a-new-branch-created-through-the-ui). This document describes the several ways to create a merge request. @@ -100,7 +100,7 @@ button to open the [**New Merge Request** page](#new-merge-request-page). A new merge request will be started using the current branch as the source, and the default branch in the current project as the target. -## New merge request from you local environment +## New merge request from your local environment Assuming you have your repository cloned into your computer and you'd like to start working on changes to files, start by creating and diff --git a/doc/user/project/merge_requests/getting_started.md b/doc/user/project/merge_requests/getting_started.md index 0ab8d31403e..32eb0c73ed4 100644 --- a/doc/user/project/merge_requests/getting_started.md +++ b/doc/user/project/merge_requests/getting_started.md @@ -67,7 +67,7 @@ Once you have created the merge request, you can also: - Preview continuous integration [pipelines on the merge request widget](reviewing_and_managing_merge_requests.md#pipeline-status-in-merge-requests-widgets). - Preview how your changes look directly on your deployed application with [Review Apps](reviewing_and_managing_merge_requests.md#live-preview-with-review-apps). - [Allow collaboration on merge requests across forks](allow_collaboration.md). -- Perform a [Review](../../discussions/index.md#merge-request-reviews-premium) in order to create multiple comments on a diff and publish them once you're ready. **(PREMIUM)** +- Perform a [Review](../../discussions/index.md#merge-request-reviews) in order to create multiple comments on a diff and publish them once you're ready. - Add [code suggestions](../../discussions/index.md#suggest-changes) to change the content of merge requests directly into merge request threads, and easily apply them to the codebase directly from the UI. - Add a time estimation and the time spent with that merge request with [Time Tracking](../time_tracking.md#time-tracking). @@ -86,7 +86,7 @@ and the merge request will be added to their #### Multiple assignees **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2004) in [GitLab Starter 11.11](https://about.gitlab.com/pricing/). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2004) in [GitLab Starter 11.11](https://about.gitlab.com/pricing/). Multiple people often review merge requests at the same time. GitLab allows you to have multiple assignees for merge requests diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 2038414dab8..5d2813f5bfc 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -54,7 +54,7 @@ To get started, read the [introduction to merge requests](getting_started.md). ## Merge request navigation tabs at the top -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/33813) in GitLab 12.6. This positioning is experimental. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33813) in GitLab 12.6. This positioning is experimental. So far, the navigation tabs present in merge requests to display **Discussion**, **Commits**, **Pipelines**, and **Changes** were located after the merge request @@ -86,35 +86,7 @@ See the features at your disposal to [review and manage merge requests](reviewin ## Testing and reports in merge requests -GitLab has the ability to test the changes included in a merge request, and can display -or link to useful information directly in the merge request page: - -| Feature | Description | -|--------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Accessibility Testing](accessibility_testing.md) | Automatically report A11y violations for changed pages in merge requests | -| [Browser Performance Testing](browser_performance_testing.md) **(PREMIUM)** | Quickly determine the performance impact of pending code changes. | -| [Code Quality](code_quality.md) **(STARTER)** | Analyze your source code quality using the [Code Climate](https://codeclimate.com/) analyzer and show the Code Climate report right in the merge request widget area. | -| [Display arbitrary job artifacts](../../../ci/yaml/README.md#artifactsexpose_as) | Configure CI pipelines with the `artifacts:expose_as` parameter to directly link to selected [artifacts](../../../ci/pipelines/job_artifacts.md) in merge requests. | -| [GitLab CI/CD](../../../ci/README.md) | Build, test, and deploy your code in a per-branch basis with built-in CI/CD. | -| [JUnit test reports](../../../ci/junit_test_reports.md) | Configure your CI jobs to use JUnit test reports, and let GitLab display a report on the merge request so that it’s easier and faster to identify the failure without having to check the entire job log. | -| [License Compliance](../../compliance/license_compliance/index.md) **(ULTIMATE)** | Manage the licenses of your dependencies. | -| [Metrics Reports](../../../ci/metrics_reports.md) **(PREMIUM)** | Display the Metrics Report on the merge request so that it's fast and easy to identify changes to important metrics. | -| [Multi-Project pipelines](../../../ci/multi_project_pipelines.md) **(PREMIUM)** | When you set up GitLab CI/CD across multiple projects, you can visualize the entire pipeline, including all cross-project interdependencies. | -| [Pipelines for merge requests](../../../ci/merge_request_pipelines/index.md) | Customize a specific pipeline structure for merge requests in order to speed the cycle up by running only important jobs. | -| [Pipeline Graphs](../../../ci/pipelines/index.md#visualize-pipelines) | View the status of pipelines within the merge request, including the deployment process. | -| [Test Coverage visualization](test_coverage_visualization.md) | See test coverage results for merge requests, within the file diff. | - -### Security Reports **(ULTIMATE)** - -In addition to the reports listed above, GitLab can do many types of [Security reports](../../application_security/index.md), -generated by scanning and reporting any vulnerabilities found in your project: - -| Feature | Description | -|-----------------------------------------------------------------------------------------|------------------------------------------------------------------| -| [Container Scanning](../../application_security/container_scanning/index.md) | Analyze your Docker images for known vulnerabilities. | -| [Dynamic Application Security Testing (DAST)](../../application_security/dast/index.md) | Analyze your running web applications for known vulnerabilities. | -| [Dependency Scanning](../../application_security/dependency_scanning/index.md) | Analyze your dependencies for known vulnerabilities. | -| [Static Application Security Testing (SAST)](../../application_security/sast/index.md) | Analyze your source code for known vulnerabilities. | +Learn about the options for [testing and reports](testing_and_reports_in_merge_requests.md) on the changes in a merge request. ## Authorization for merge requests diff --git a/doc/user/project/merge_requests/merge_request_approvals.md b/doc/user/project/merge_requests/merge_request_approvals.md index e5896e62397..dd90449cd86 100644 --- a/doc/user/project/merge_requests/merge_request_approvals.md +++ b/doc/user/project/merge_requests/merge_request_approvals.md @@ -34,10 +34,12 @@ minimum number of required approvers can still be set in the [project settings f ### Eligible approvers -The following can approve merge requests: +The following users can approve merge requests: -- Users being added as approvers at project or merge request level. -- [Code owners](#code-owners-as-eligible-approvers) to the files changed by the merge request. +- Users who have been added as approvers at the project or merge request levels with + developer or higher [permissions](../../permissions.md). +- [Code owners](#code-owners-as-eligible-approvers) of the files changed by the merge request + that have developer or higher [permissions](../../permissions.md). An individual user can be added as an approver for a project if they are a member of: @@ -46,7 +48,7 @@ An individual user can be added as an approver for a project if they are a membe - A group that has access to the project via a [share](../members/share_project_with_groups.md). A group of users can also be added as approvers. In the future, group approvers may be -[restricted to only groups with share access to the project](https://gitlab.com/gitlab-org/gitlab/issues/2048). +[restricted to only groups with share access to the project](https://gitlab.com/gitlab-org/gitlab/-/issues/2048). If a user is added as an individual approver and is also part of a group approver, then that user is just counted once. The merge request author, as well as users who have committed @@ -68,7 +70,7 @@ were not explicitly listed in the approval rules. If you add [Code Owners](../code_owners.md) to your repository, the owners to the corresponding files will become eligible approvers, together with members with Developer -or higher permissions. +or higher [permissions](../../permissions.md). To enable this merge request approval rule: @@ -127,7 +129,7 @@ the same steps as [Adding / editing a default approval rule](#adding--editing-a- ### Multiple approval rules **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1979) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1979) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. In GitLab Premium, it is possible to have multiple approval rules per merge request, as well as multiple default approval rules per project. @@ -149,7 +151,7 @@ reduce the number of approvals left for all rules that the approver belongs to. ### Scoped to Protected Branch **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/460) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8. Approval rules are often only relevant to specific branches, like `master`. When configuring [**Default Approval Rules**](#adding--editing-a-default-approval-rule) @@ -222,7 +224,7 @@ from the UI. However, approvals will be reset if the target branch is changed. ### Allowing merge request authors to approve their own merge requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3349) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3349) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.3. You can allow merge request authors to self-approve merge requests. Authors also need to be included in the approvers list in order to be able to @@ -234,7 +236,7 @@ approve their merge request. To enable this feature: ### Prevent approval of merge requests by their committers -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/10441) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10441) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.10. You can prevent users that have committed to a merge request from approving it. To enable this feature: @@ -244,7 +246,12 @@ enable this feature: ### Require authentication when approving a merge request -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5981) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5981) in [GitLab Starter](https://about.gitlab.com/pricing/) 12.0. + +NOTE: **Note:** +To require authentication when approving a merge request, you must enable +**Password authentication enabled for web interface** under [sign-in restrictions](../../admin_area/settings/sign_in_restrictions.md#password-authentication-enabled). +in the Admin area. You can force the approver to enter a password in order to authenticate before adding the approval. This enables an Electronic Signature for approvals such as the one defined @@ -264,7 +271,7 @@ For more information, see ## Enabling the new approvals interface -Since [GitLab v12.0](https://gitlab.com/gitlab-org/gitlab/issues/10685), an updated approvals +Since [GitLab v12.0](https://gitlab.com/gitlab-org/gitlab/-/issues/10685), an updated approvals interface is available by default. In versions older than 12.0, the updated interface is not available unless the `approval_rules` feature flag is enabled, which can be done from the Rails console by instance administrators. diff --git a/doc/user/project/merge_requests/merge_request_dependencies.md b/doc/user/project/merge_requests/merge_request_dependencies.md index 66a1d2ac6af..a1012e27d32 100644 --- a/doc/user/project/merge_requests/merge_request_dependencies.md +++ b/doc/user/project/merge_requests/merge_request_dependencies.md @@ -4,7 +4,7 @@ type: reference, concepts # Merge Request dependencies **(PREMIUM)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9688) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9688) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. > - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17291) from "Cross-project dependencies" to "Merge Requests dependencies" in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. > - Intra-project MR dependencies were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16799) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. @@ -94,9 +94,9 @@ merge. ## Limitations -- API support: [issue #12551](https://gitlab.com/gitlab-org/gitlab/issues/12551) -- Dependencies are not preserved across project export/import: [issue #12549](https://gitlab.com/gitlab-org/gitlab/issues/12549) -- Complex merge order dependencies are not supported: [issue #11393](https://gitlab.com/gitlab-org/gitlab/issues/11393) +- API support: [issue #12551](https://gitlab.com/gitlab-org/gitlab/-/issues/12551) +- Dependencies are not preserved across project export/import: [issue #12549](https://gitlab.com/gitlab-org/gitlab/-/issues/12549) +- Complex merge order dependencies are not supported: [issue #11393](https://gitlab.com/gitlab-org/gitlab/-/issues/11393) The last item merits a little more explanation. Dependencies between merge requests can be described as a graph of relationships. The simplest possible diff --git a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md index 50412e0b319..d45ccdc9be9 100644 --- a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md +++ b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md @@ -59,11 +59,24 @@ merge request from the UI, until you make all relevant jobs pass.  +### Skipped pipelines + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211482) in GitLab 13.1. + +When the **Pipelines must succeed** checkbox is checked, [skipped pipelines](../../../ci/yaml/README.md#skip-pipeline) prevent +merge requests from being merged. To change this behavior: + +1. Navigate to your project's **Settings > General** page. +1. Expand the **Merge requests** section. +1. In the **Merge checks** subsection, ensure **Pipelines must succeed** is checked. +1. In the **Merge checks** subsection, select the **Skipped pipelines are considered successful** checkbox. +1. Press **Save** for the changes to take effect. + ### Limitations When this setting is enabled, a merge request is prevented from being merged if there is no pipeline. This may conflict with some use cases where [`only/except`](../../../ci/yaml/README.md#onlyexcept-advanced) rules are used and they don't generate any pipelines. -Users that expect to be able to merge a merge request in this scenario should ensure that [there is always a pipeline](https://gitlab.com/gitlab-org/gitlab-foss/issues/54226) and that it's successful. +Users that expect to be able to merge a merge request in this scenario should ensure that [there is always a pipeline](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/54226) and that it's successful. For example, to that on merge requests there is always a passing job even though `only/except` rules may not generate any other jobs: diff --git a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md index fdcb1049ef7..a3ad41d8dd8 100644 --- a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md +++ b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md @@ -82,6 +82,10 @@ to expand the entire file.  +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205401) in GitLab 13.1, when viewing a +merge request's **Changes** tab, if a certain file was only renamed, you can expand it to see the +entire content by clicking **Show file contents**. + ### Ignore whitespace changes in Merge Request diff view If you click the **Hide whitespace changes** button, you can see the diff @@ -96,7 +100,7 @@ whitespace changes. ## Perform inline code reviews -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/13950) in GitLab 11.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/13950) in GitLab 11.5. GitLab provides a way of leaving comments in any part of the file being changed in a Merge Request. To do so, click the **...** button in the gutter of the Merge Request diff UI to expand the diff lines and leave a comment, just as you would for a changed line. @@ -108,7 +112,7 @@ in a Merge Request. To do so, click the **...** button in the gutter of the Merg If you've set up [GitLab CI/CD](../../../ci/README.md) in your project, you will be able to see: -- Both pre and post-merge pipelines and the environment information if any. +- Both pre-merge and post-merge pipelines and the environment information if any. - Which deployments are in progress. If there's an [environment](../../../ci/environments/index.md) and the application is diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index 84d60fca72d..1c0e698aba5 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -4,7 +4,7 @@ type: reference, howto # Test Coverage Visualization -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/3708) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3708) in GitLab 12.9. With the help of [GitLab CI/CD](../../../ci/README.md), you can collect the test coverage information of your favorite testing or coverage-analysis tool, and visualize @@ -67,7 +67,7 @@ test: This feature comes with the `:coverage_report_view` feature flag disabled by default. This feature is disabled due to some performance issues with very large -data sets. When [the performance issue](https://gitlab.com/gitlab-org/gitlab/issues/211410) +data sets. When [the performance issue](https://gitlab.com/gitlab-org/gitlab/-/issues/211410) is resolved, the feature will be enabled by default. To enable this feature, ask a GitLab administrator with Rails console access to diff --git a/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md b/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md new file mode 100644 index 00000000000..f7614ed7996 --- /dev/null +++ b/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md @@ -0,0 +1,36 @@ +--- +type: index +description: "Test your code and display reports in merge requests" +--- + +# Tests and reports in merge requests + +GitLab has the ability to test the changes included in a feature branch and display reports +or link to useful information directly from merge requests: + +| Feature | Description | +|--------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Accessibility Testing](accessibility_testing.md) | Automatically report A11y violations for changed pages in merge requests. | +| [Browser Performance Testing](browser_performance_testing.md) **(PREMIUM)** | Quickly determine the performance impact of pending code changes. | +| [Code Quality](code_quality.md) **(STARTER)** | Analyze your source code quality using the [Code Climate](https://codeclimate.com/) analyzer and show the Code Climate report right in the merge request widget area. | +| [Display arbitrary job artifacts](../../../ci/yaml/README.md#artifactsexpose_as) | Configure CI pipelines with the `artifacts:expose_as` parameter to directly link to selected [artifacts](../../../ci/pipelines/job_artifacts.md) in merge requests. | +| [GitLab CI/CD](../../../ci/README.md) | Build, test, and deploy your code in a per-branch basis with built-in CI/CD. | +| [JUnit test reports](../../../ci/junit_test_reports.md) | Configure your CI jobs to use JUnit test reports, and let GitLab display a report on the merge request so that it’s easier and faster to identify the failure without having to check the entire job log. | +| [License Compliance](../../compliance/license_compliance/index.md) **(ULTIMATE)** | Manage the licenses of your dependencies. | +| [Metrics Reports](../../../ci/metrics_reports.md) **(PREMIUM)** | Display the Metrics Report on the merge request so that it's fast and easy to identify changes to important metrics. | +| [Multi-Project pipelines](../../../ci/multi_project_pipelines.md) **(PREMIUM)** | When you set up GitLab CI/CD across multiple projects, you can visualize the entire pipeline, including all cross-project interdependencies. | +| [Pipelines for merge requests](../../../ci/merge_request_pipelines/index.md) | Customize a specific pipeline structure for merge requests in order to speed the cycle up by running only important jobs. | +| [Pipeline Graphs](../../../ci/pipelines/index.md#visualize-pipelines) | View the status of pipelines within the merge request, including the deployment process. | +| [Test Coverage visualization](test_coverage_visualization.md) | See test coverage results for merge requests, within the file diff. | + +## Security Reports **(ULTIMATE)** + +In addition to the reports listed above, GitLab can do many types of [Security reports](../../application_security/index.md), +generated by scanning and reporting any vulnerabilities found in your project: + +| Feature | Description | +|-----------------------------------------------------------------------------------------|------------------------------------------------------------------| +| [Container Scanning](../../application_security/container_scanning/index.md) | Analyze your Docker images for known vulnerabilities. | +| [Dynamic Application Security Testing (DAST)](../../application_security/dast/index.md) | Analyze your running web applications for known vulnerabilities. | +| [Dependency Scanning](../../application_security/dependency_scanning/index.md) | Analyze your dependencies for known vulnerabilities. | +| [Static Application Security Testing (SAST)](../../application_security/sast/index.md) | Analyze your source code for known vulnerabilities. | diff --git a/doc/user/project/merge_requests/versions.md b/doc/user/project/merge_requests/versions.md index 84934148bdc..9581c974414 100644 --- a/doc/user/project/merge_requests/versions.md +++ b/doc/user/project/merge_requests/versions.md @@ -37,7 +37,7 @@ changes appears as a system note. ## Find the merge request that introduced a change -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/2383) in GitLab 10.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/2383) in GitLab 10.5. When viewing the commit details page, GitLab will link to the merge request (or merge requests, if it's in more than one) containing that commit. @@ -48,7 +48,7 @@ request, they will not be linked. ## `HEAD` comparison mode for Merge Requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27008) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27008) in GitLab 12.10. Merge Requests, particularly the **Changes** tab, is where source code is reviewed and discussed. In circumstances where the target branch was @@ -62,7 +62,7 @@ shows a diff calculated by simulating how it would look like once merged - a mor representation of the changes rather than using the base of the two branches. The new mode is available from the comparison target drop down by selecting **master (HEAD)**. In the future it will -[replace](https://gitlab.com/gitlab-org/gitlab/issues/198458) the +[replace](https://gitlab.com/gitlab-org/gitlab/-/issues/198458) the current default comparison.  diff --git a/doc/user/project/milestones/burndown_charts.md b/doc/user/project/milestones/burndown_charts.md index e9f23899068..e28ba1d2d5f 100644 --- a/doc/user/project/milestones/burndown_charts.md +++ b/doc/user/project/milestones/burndown_charts.md @@ -1,5 +1,8 @@ --- type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Burndown Charts **(STARTER)** diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index 085c1bd143e..421cb42de1e 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -1,5 +1,8 @@ --- type: index, reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Milestones diff --git a/doc/user/project/new_ci_build_permissions_model.md b/doc/user/project/new_ci_build_permissions_model.md index 120c7a35cb2..cfcbf11a407 100644 --- a/doc/user/project/new_ci_build_permissions_model.md +++ b/doc/user/project/new_ci_build_permissions_model.md @@ -1,10 +1,17 @@ +--- +stage: Verify +group: Continuous Integration +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: reference +--- + # New CI job permissions model > Introduced in GitLab 8.12. GitLab 8.12 has a completely redesigned [job permissions](../permissions.md#job-permissions) system. You can find all discussion and all our concerns when choosing the current approach in issue -[#18994](https://gitlab.com/gitlab-org/gitlab-foss/issues/18994). +[#18994](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/18994). Jobs permissions should be tightly integrated with the permissions of a user who is triggering a job. @@ -68,7 +75,7 @@ Let's consider the following scenario: A unique job token is generated for each job and provides the user read access all projects that would be normally accessible to the user creating that job. The unique job token does not have any write permissions, but there -is a [proposal to add support](https://gitlab.com/gitlab-org/gitlab/issues/35067). +is a [proposal to add support](https://gitlab.com/gitlab-org/gitlab/-/issues/35067). We try to make sure that this token doesn't leak by: @@ -156,7 +163,7 @@ As an administrator: [check manually](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/doc/update) for installations from source. - **500 errors**: Check if you have another web proxy sitting in front of NGINX (HAProxy, Apache, etc.). It might be a good idea to let GitLab use the internal NGINX - web server and not disable it completely. See [this comment](https://gitlab.com/gitlab-org/gitlab-foss/issues/22484#note_16648302) for an + web server and not disable it completely. See [this comment](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22484#note_16648302) for an example. - **403 errors**: You need to make sure that your installation has [HTTP(S) cloning enabled](../admin_area/settings/visibility_and_access_controls.md#enabled-git-access-protocols). HTTP(S) support is now a **requirement** by GitLab CI @@ -178,7 +185,7 @@ git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.com/<user>/<mydependent ``` It can also be used for system-wide authentication -(only do this in a docker container, it will overwrite ~/.netrc): +(only do this in a Docker container, it will overwrite ~/.netrc): ```shell echo -e "machine gitlab.com\nlogin gitlab-ci-token\npassword ${CI_JOB_TOKEN}" > ~/.netrc diff --git a/doc/user/project/operations/alert_management.md b/doc/user/project/operations/alert_management.md index 2dcf72aaf01..411b36411af 100644 --- a/doc/user/project/operations/alert_management.md +++ b/doc/user/project/operations/alert_management.md @@ -1,3 +1,9 @@ +--- +stage: Monitor +group: Health +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Alert Management > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2877) in GitLab 13.0. @@ -32,12 +38,14 @@ immediately identify which alerts you should prioritize investigating: Alerts contain one of the following icons: -- **Critical**: **{severity-critical}** and hexadecimal color `#8b2615` -- **High**: **{severity-high}** and hexadecimal color `#c0341d` -- **Medium**: **{severity-medium}** and hexadecimal color `#fca429` -- **Low**: **{severity-low}** and hexadecimal color `#fdbc60` -- **Info**: **{severity-info}** and hexadecimal color `#418cd8` -- **Unknown**: **{severity-unknown}** and hexadecimal color `#bababa` +| Severity | Icon | Color (hexadecimal) | +|---|---|---| +| Critical | **{severity-critical}** | `#8b2615` | +| High | **{severity-high}** | `#c0341d` | +| Medium | **{severity-medium}** | `#fca429` | +| Low | **{severity-low}** | `#fdbc60` | +| Info | **{severity-info}** | `#418cd8` | +| Unknown | **{severity-unknown}** | `#bababa` | ## Alert Management list @@ -72,8 +80,91 @@ You will need at least Developer [permissions](../../permissions.md) to view Ale Navigate to the Alert Management detail view by visiting the [Alert Management list](#alert-management-list) and selecting an Alert from the list. - + + + ### Update an Alert's status -The Alert Management detail view allows users to update the Alert Status. See [Alert Management statuses](#alert-management-statuses) for more details. +The Alert Management detail view enables you to update the Alert Status. +See [Alert Management statuses](#alert-management-statuses) for more details. + +### Create an Issue from an Alert + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. + +The Alert Management detail view enables you to create an issue with a +description automatically populated from an alert. To create the issue, +click the **Create Issue** button. You can then view the issue from the +alert by clicking the **View Issue** button. + +Closing a GitLab issue associated with an alert changes the alert's status to Resolved. +See [Alert Management statuses](#alert-management-statuses) for more details about statuses. + +### Update an Alert's assignee + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. + +The Alert Management detail view allows users to update the Alert assignee. + +In large teams, where there is shared ownership of an alert, it can be difficult +to track who is investigating and working on it. The Alert Management detail view +enables you to update the Alert assignee: + +NOTE: **Note:** +GitLab currently only supports a single assignee per alert. + +1. To display the list of current alerts, click + **{cloud-gear}** **Operations > Alerts**: + +  + +1. Select your desired alert to display its **Alert Management Details View**: + +  + +1. If the right sidebar is not expanded, click + **{angle-double-right}** **Expand sidebar** to expand it. +1. In the right sidebar, locate the **Assignee** and click **Edit**. From the + dropdown menu, select each user you want to assign to the alert. GitLab creates + a [To-Do list item](../../todos.md) for each user. + +  + +To remove an assignee, click **Edit** next to the **Assignee** dropdown menu and +deselect the user from the list of assignees, or click **Unassigned**. + +### Alert system notes + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. + +NOTE: **Note:** +GitLab currently only supports creating system notes when assigning an Alert. + +Assigning a user an Alert creates a system note, visible in the Alert Details view, +giving you a linear timeline of the alert's investigation and assignment history. + + + +## Use cases for assigning alerts + +Consider a team formed by different sections of monitoring, collaborating on a +single application. After an alert surfaces, it's extremely important to +route the alert to the team members who can address and resolve the alert. + +Assigning Alerts to multiple assignees eases collaboration and delegation. All +assignees are shown in your team's work-flows, and all assignees receive +notifications, simplifying communication and ownership of the alert. + +After completing their portion of investigating or fixing the alert, users can +unassign their account from the alert when their role is complete. +The [alerts status](#alert-management-statuses) can be updated to +reflect if the alert has been resolved. + +### Slack Notifications + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216326) in GitLab 13.1. + +You can be alerted via a Slack message when a new alert has been received. + +See the [Slack Notifications Service docs](../integrations/slack.md) for information on how to set this up. diff --git a/doc/user/project/operations/dashboard_settings.md b/doc/user/project/operations/dashboard_settings.md new file mode 100644 index 00000000000..e14c478ab7a --- /dev/null +++ b/doc/user/project/operations/dashboard_settings.md @@ -0,0 +1,47 @@ +--- +stage: Monitor +group: APM +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Metrics dashboard settings + +You can configure your [Monitoring dashboard](../integrations/prometheus.md) to +display the time zone of your choice, and the links of your choice. + +## Change the dashboard time zone + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214370) in GitLab 13.1. + +By default, your monitoring dashboard displays dates and times in your local +time zone, but you can display dates and times in UTC format. To change the +time zone: + +1. Navigate to **{settings}** **Settings > Operations**, and scroll to + **Metrics Dashboard**. +1. In the **Dashboard timezone** select box, select *User's local timezone* + or *UTC*: + +  +1. Click **Save changes**. + +## Link to an external dashboard + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57171) in GitLab 12.0. + +You can add a button on your monitoring dashboard that links directly to your +existing external dashboards: + +1. Navigate to **{settings}** **Settings > Operations**, and scroll to + **Metrics Dashboard**. +1. In **External dashboard URL**, provide the URL to your external dashboard: + +  + +1. Click **Save changes**. + +GitLab displays a **View full dashboard** button in the top right corner of your +[monitoring dashboard](../../../ci/environments/index.md#monitoring-environments) +which opens the URL you provided: + + diff --git a/doc/user/project/operations/error_tracking.md b/doc/user/project/operations/error_tracking.md index 23a50fd7766..a7a16de90e0 100644 --- a/doc/user/project/operations/error_tracking.md +++ b/doc/user/project/operations/error_tracking.md @@ -61,7 +61,7 @@ This page has: - A link to the Sentry issue. - A link to the GitLab commit if the Sentry [release ID/version](https://docs.sentry.io/workflow/releases/?platform=javascript#configure-sdk) on the Sentry Issue's first release matches a commit SHA in your GitLab hosted project. - Other details about the issue, including a full stack trace. -- In [GitLab 12.7 and newer](https://gitlab.com/gitlab-org/gitlab/issues/36246), language and urgency are displayed. +- In [GitLab 12.7 and newer](https://gitlab.com/gitlab-org/gitlab/-/issues/36246), language and urgency are displayed. By default, a **Create issue** button is displayed: @@ -77,7 +77,7 @@ You can take action on Sentry Errors from within the GitLab UI. ### Ignoring errors -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/39665) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39665) in GitLab 12.7. From within the [Error Details](#error-details) page you can ignore a Sentry error by simply clicking the **Ignore** button near the top of the page. @@ -85,7 +85,7 @@ Ignoring an error will prevent it from appearing in the [Error Tracking List](#e ### Resolving errors -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/39825) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39825) in GitLab 12.7. From within the [Error Details](#error-details) page you can resolve a Sentry error by clicking the **Resolve** button near the top of the page. diff --git a/doc/user/project/operations/feature_flags.md b/doc/user/project/operations/feature_flags.md index 8716d5feb4a..a9729204cd7 100644 --- a/doc/user/project/operations/feature_flags.md +++ b/doc/user/project/operations/feature_flags.md @@ -8,100 +8,73 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7433) in GitLab 11.4. -Feature flags allow you to ship a project in different flavors by -dynamically toggling certain functionality. +With Feature Flags, you can deploy your application's new features to production in smaller batches. +You can toggle a feature on and off to subsets of users, helping you achieve Continuous Delivery. +Feature flags help reduce risk, allowing you to do controlled testing, and separate feature +delivery from customer launch. -## Overview - -Feature Flags offer a feature toggle system for your application. They enable teams -to achieve Continuous Delivery by deploying new features to production at smaller -batches for controlled testing, separating feature delivery from customer launch. -This helps reducing risk and allows you to easily manage which features to enable. - -GitLab offers a Feature Flags interface that allows you to create, toggle and -remove feature flags. - -<div class="video-fallback"> - <a href="https://www.youtube.com/watch?v=5tw2p6lwXxo">Watch</a> a use case between Feature Flags and Sentry Error Tracking -</div> -<figure class="video-container"> - <iframe src="https://www.youtube.com/embed/5tw2p6lwXxo" frameborder="0" allowfullscreen="true"> </iframe> -</figure> +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an example of feature flags in action, see [GitLab for Deploys, Feature Flags, and Error Tracking](https://www.youtube.com/embed/5tw2p6lwXxo). ## How it works -Underneath, GitLab uses [unleash](https://github.com/Unleash/unleash), a feature -toggle service. GitLab provides an API where your application can talk to and get the -list of feature flags you set in GitLab. +GitLab uses [Unleash](https://github.com/Unleash/unleash), a feature +toggle service. -The application must be configured to talk to GitLab, so that's up to the -developers to use a compatible [client library](#client-libraries) and -integrate it in their app. +By enabling or disabling a flag in GitLab, your application +can determine which features to enable or disable. -By setting a flag active or inactive via GitLab, your application will automatically -know which features to enable or disable respectively. +You can create feature flags in GitLab and use the API from your application +to get the list of feature flags and their statuses. The application must be configured to communicate +with GitLab, so it's up to developers to use a compatible client library and +[integrate the feature flags in your app](#integrate-feature-flags-with-your-application). -## Adding a new feature flag +## Create a feature flag -To add a new feature flag: +To create and enable a feature flag: 1. Navigate to your project's **Operations > Feature Flags**. -1. Click on the **New Feature Flag** button. -1. Give it a name. - - NOTE: **Note:** - A name can contain only lowercase letters, digits, underscores (`_`) - and dashes (`-`), must start with a letter, and cannot end with a dash (`-`) - or an underscore (`_`). - -1. Give it a description (optional, 255 characters max). -1. Define environment [specs](#define-environment-specs). If you want the flag on by default, enable the catch-all [wildcard spec (`*`)](#define-environment-specs) +1. Click the **New feature flag** button. +1. Enter a name that starts with a letter and contains only lowercase letters, digits, underscores (`_`) + and dashes (`-`), and does not end with a dash (`-`) or underscore (`_`). +1. Enter a description (optional, 255 characters max). +1. Enter details about how the flag should be applied: + - In GitLab 13.0 and earlier: Enter an environment spec, + enable or disable the flag in this environment, and select a rollout strategy. + - In GitLab 13.1 and later (when [this feature flag](#feature-flag-behavior-change-in-130) is enabled): Select a strategy and then + the environments to apply the strategy to. 1. Click **Create feature flag**. -Once a feature flag is created, the list of existing feature flags will be presented -with ability to edit or remove them. - -To make a feature flag active or inactive, click the pencil icon to edit it, -and toggle the status for each [spec](#define-environment-specs). +The feature flag is displayed in the list. It is enabled by default. -The toggles next to each feature flag on the list page function as global shutoff switches. -If a toggle is off, that feature flag is disabled for every environment. +## Disable a feature flag for a specific environment - +In [GitLab 13.0 and earlier](https://gitlab.com/gitlab-org/gitlab/-/issues/8621), +to disable a feature flag for a specific environment: -## Define environment specs - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8621) in GitLab 11.8. - -In general, an application is deployed to multiple environments, such as -production, staging and [review apps](../../../ci/review_apps/index.md). -For example, you may not want to enable a feature flag on production until your QA team has -first confirmed that the feature is working correctly on testing environments. +1. Navigate to your project's **Operations > Feature Flags**. +1. For the feature flag you want to disable, click the Pencil icon. +1. To disable the flag: + - In GitLab 13.0 and earlier: Slide the Status toggle for the environment. Or, to delete the + environment spec, on the right, click the **Remove (X)** icon. + - In GitLab 13.1 and later (when [this feature flag](#feature-flag-behavior-change-in-130) is + enabled): For each strategy it applies to, under **Environments**, delete the environment. +1. Click **Save changes**. -To handle these situations, you can enable a feature flag on a particular environment -with [Environment specs](../../../ci/environments/index.md#scoping-environments-with-specs). -You can define multiple specs per flag so that you can control your feature flag more granularly. +## Disable a feature flag for all environments -To define specs for each environment: +To disable a feature flag for all environments: 1. Navigate to your project's **Operations > Feature Flags**. -1. Click on the **New Feature Flag** button or edit an existing flag. -1. Set the status of the default [spec](../../../ci/environments/index.md#scoping-environments-with-specs) (`*`). Choose a rollout strategy. This status and rollout strategy combination will be used for _all_ environments. -1. If you want to enable/disable the feature on a specific environment, create a new [spec](../../../ci/environments/index.md#scoping-environments-with-specs) and type the environment name. -1. Set the status and rollout strategy of the additional spec. This status and rollout strategy combination takes precedence over the default spec since we always use the most specific match available. -1. Click **Create feature flag** or **Update feature flag**. +1. For the feature flag you want to disable, slide the Status toggle to **Disabled**. - - -NOTE: **NOTE** -We'd highly recommend you to use the [Environment](../../../ci/environments/index.md) -feature in order to quickly assess which flag is enabled per environment. +The feature flag is displayed on the **Disabled** tab. ## Feature flag behavior change in 13.0 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35555) in GitLab 13.0. -Starting in GitLab 13.0, you can apply a feature flag strategy across multiple environment specs, +Starting in GitLab 13.0, you can apply a feature flag strategy across multiple environments, without defining the strategy multiple times. This feature is under development and not ready for production use. It is @@ -121,29 +94,18 @@ To disable it: Feature.disable(:feature_flags_new_version) ``` -### Applying a strategy to environments - -After a strategy is defined, it applies to **All Environments** by default. To -make a strategy apply to a specific environment spec: +## Feature flag strategies -1. Click the **Add Environment** button. -1. Create a new - [spec](../../../ci/environments/index.md#scoping-environments-with-specs). - -To apply the strategy to multiple environment specs, repeat these steps. - -## Feature Flag strategies - -GitLab Feature Flag adopts [Unleash](https://unleash.github.io) -as the feature flag engine. In unleash, there is a concept of rulesets for granular feature flag controls, +GitLab Feature Flag uses [Unleash](https://unleash.github.io) +as the feature flag engine. In Unleash, there is a concept of rulesets for granular feature flag controls, called [strategies](https://unleash.github.io/docs/activation_strategy). Supported strategies for GitLab Feature Flags are described below. ### Rollout strategy -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8240) in GitLab 12.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. -The selected rollout strategy affects which users will experience the feature enabled. +The selected rollout strategy affects which users will experience the feature as enabled. The status of an environment spec ultimately determines whether or not a feature is enabled at all. For instance, a feature will always be disabled for every user if the matching environment spec has a disabled status, regardless of the chosen rollout strategy. @@ -172,36 +134,34 @@ ID for the feature to be enabled. See the [Ruby example](#ruby-application-examp #### User IDs -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/8240) in GitLab 12.2. [Updated](https://gitlab.com/gitlab-org/gitlab/issues/34363) to be defined per environment in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8240) in GitLab 12.2. [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/34363) to be defined per environment in GitLab 12.6. A feature flag may be enabled for a list of target users. It is implemented using the Unleash [`userWithId`](https://unleash.github.io/docs/activation_strategy#userwithid) activation strategy. -User IDs should be a comma separated list of values. For example, `user@example.com, user2@example.com`, or `username1,username2,username3`, etc. +User IDs should be a comma-separated list of values. For example, `user@example.com, user2@example.com`, or `username1,username2,username3`, etc. CAUTION: **Caution:** The Unleash client **must** be given a user ID for the feature to be enabled for target users. See the [Ruby example](#ruby-application-example) below. -## Integrating with your application +## Integrate feature flags with your application -In order to use Feature Flags, you need to first -[get the access credentials](#configuring-feature-flags) from GitLab and then -prepare your application and hook it with a [client library](#client-libraries). +To use feature flags with your application, get access credentials from GitLab. +Then prepare your application with a client library. -## Configuring Feature Flags +### Get access credentials -To get the access credentials that your application will need to talk to GitLab: +To get the access credentials that your application needs to communicate with GitLab: 1. Navigate to your project's **Operations > Feature Flags**. -1. Click on the **Configure** button to see the values: +1. Click the **Configure** button to view the following: - **API URL**: URL where the client (application) connects to get a list of feature flags. - **Instance ID**: Unique token that authorizes the retrieval of the feature flags. - **Application name**: The name of the running environment. For instance, - if the application runs for a production server, application name would be - `production` or similar. This value is used for - [the environment spec evaluation](#define-environment-specs). + if the application runs for a production server, the application name would be + `production` or similar. This value is used for the environment spec evaluation. NOTE: **Note:** The meaning of these fields might change over time. For example, we are not sure @@ -209,34 +169,28 @@ if **Instance ID** will be single token or multiple tokens, assigned to the **Environment**. Also, **Application name** could describe the version of application instead of the running environment. -## Client libraries +### Choose a client library -GitLab currently implements a single backend that is compatible with -[Unleash](https://github.com/Unleash/unleash#client-implementations) clients. +GitLab implements a single backend that is compatible with Unleash clients. -Unleash clients allow the developers to define in the app's code the default -values for flags. Each feature flag evaluation can express the desired -outcome in case the flag isn't present on the provided configuration file. +With the Unleash client, developers can define, in the application code, the default values for flags. +Each feature flag evaluation can express the desired outcome if the flag isn't present in the +provided configuration file. -Unleash currently offers a number of official SDKs for various frameworks and -a number of community contributed libraries. +Unleash currently [offers many SDKs for various languages and frameworks](https://github.com/Unleash/unleash#client-implementations). -Official clients: +### Feature flags API information -- [Unleash client SDK for Java](https://github.com/unleash/unleash-client-java) -- [Unleash client SDK for Node.js](https://github.com/unleash/unleash-client-node) -- [Unleash client for Go](https://github.com/unleash/unleash-client-go) -- [Unleash client for Ruby](https://github.com/unleash/unleash-client-ruby) +For API content, see: -Community contributed clients: - -- [Unleash FeatureToggle Client for .Net](https://github.com/stiano/unleash-client-dotnet) -- [Unofficial .Net Core Unleash client](https://github.com/onybo/unleash-client-core) -- [Unleash client for Python 3](https://github.com/aes/unleash-client-python) +- [Feature Flags API](../../../api/feature_flags.md) +- [Feature Flag Specs API](../../../api/feature_flag_specs.md) (Deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369).) +- [Feature Flag User Lists API](../../../api/feature_flag_user_lists.md) +- [Legacy Feature Flags API](../../../api/feature_flags_legacy.md) -## Golang application example +### Golang application example -Here's an example of how to integrate the feature flags in a Golang application: +Here's an example of how to integrate feature flags in a Golang application: ```golang package main @@ -275,9 +229,9 @@ func main() { } ``` -## Ruby application example +### Ruby application example -Here's an example of how to integrate the feature flags in a Ruby application. +Here's an example of how to integrate feature flags in a Ruby application. The Unleash client is given a user ID for use with a **Percent rollout (logged in users)** rollout strategy or a list of **Target Users**. @@ -305,12 +259,3 @@ else puts "hello, world!" end ``` - -## Feature Flags API - -You can create, update, read, and delete Feature Flags via API -to control them in an automated flow: - -- [Feature Flags API](../../../api/feature_flags.md) -- [Feature Flag Specs API](../../../api/feature_flag_specs.md) -- [Feature Flag User Lists API](../../../api/feature_flag_user_lists.md) diff --git a/doc/user/project/operations/img/alert_detail_full_v13_1.png b/doc/user/project/operations/img/alert_detail_full_v13_1.png Binary files differnew file mode 100644 index 00000000000..18a6f4fb67b --- /dev/null +++ b/doc/user/project/operations/img/alert_detail_full_v13_1.png diff --git a/doc/user/project/operations/img/alert_detail_overview_v13_1.png b/doc/user/project/operations/img/alert_detail_overview_v13_1.png Binary files differnew file mode 100644 index 00000000000..10c945d3810 --- /dev/null +++ b/doc/user/project/operations/img/alert_detail_overview_v13_1.png diff --git a/doc/user/project/operations/img/alert_detail_system_notes_v13_1.png b/doc/user/project/operations/img/alert_detail_system_notes_v13_1.png Binary files differnew file mode 100644 index 00000000000..2a6d0320a54 --- /dev/null +++ b/doc/user/project/operations/img/alert_detail_system_notes_v13_1.png diff --git a/doc/user/project/operations/img/alert_detail_v13_0.png b/doc/user/project/operations/img/alert_detail_v13_0.png Binary files differdeleted file mode 100644 index 7da09407cd5..00000000000 --- a/doc/user/project/operations/img/alert_detail_v13_0.png +++ /dev/null diff --git a/doc/user/project/operations/img/alert_details_assignees_v13_1.png b/doc/user/project/operations/img/alert_details_assignees_v13_1.png Binary files differnew file mode 100644 index 00000000000..dab4eac384a --- /dev/null +++ b/doc/user/project/operations/img/alert_details_assignees_v13_1.png diff --git a/doc/user/project/operations/img/alert_issue_v13_1.png b/doc/user/project/operations/img/alert_issue_v13_1.png Binary files differnew file mode 100644 index 00000000000..da79074aa2f --- /dev/null +++ b/doc/user/project/operations/img/alert_issue_v13_1.png diff --git a/doc/user/project/operations/img/alert_list_assignees_v13_1.png b/doc/user/project/operations/img/alert_list_assignees_v13_1.png Binary files differnew file mode 100644 index 00000000000..db1e0d8dcb7 --- /dev/null +++ b/doc/user/project/operations/img/alert_list_assignees_v13_1.png diff --git a/doc/user/project/operations/img/alert_management_1_v13_1.png b/doc/user/project/operations/img/alert_management_1_v13_1.png Binary files differindex c01b4749eda..00aa56a6050 100644 --- a/doc/user/project/operations/img/alert_management_1_v13_1.png +++ b/doc/user/project/operations/img/alert_management_1_v13_1.png diff --git a/doc/user/project/operations/img/alert_todo_assignees_v13_1.png b/doc/user/project/operations/img/alert_todo_assignees_v13_1.png Binary files differnew file mode 100644 index 00000000000..637f8be5d25 --- /dev/null +++ b/doc/user/project/operations/img/alert_todo_assignees_v13_1.png diff --git a/doc/user/project/operations/img/dashboard_external_link_v13_1.png b/doc/user/project/operations/img/dashboard_external_link_v13_1.png Binary files differnew file mode 100644 index 00000000000..3e8d792c53e --- /dev/null +++ b/doc/user/project/operations/img/dashboard_external_link_v13_1.png diff --git a/doc/user/project/operations/img/dashboard_local_timezone_v13_1.png b/doc/user/project/operations/img/dashboard_local_timezone_v13_1.png Binary files differnew file mode 100644 index 00000000000..8d45607a940 --- /dev/null +++ b/doc/user/project/operations/img/dashboard_local_timezone_v13_1.png diff --git a/doc/user/project/operations/img/external_dashboard_settings.png b/doc/user/project/operations/img/external_dashboard_settings.png Binary files differdeleted file mode 100644 index e1b7fa56a1c..00000000000 --- a/doc/user/project/operations/img/external_dashboard_settings.png +++ /dev/null diff --git a/doc/user/project/operations/index.md b/doc/user/project/operations/index.md index 954f88fe4f2..ca38eab9455 100644 --- a/doc/user/project/operations/index.md +++ b/doc/user/project/operations/index.md @@ -10,4 +10,4 @@ your applications: - Discover and view errors generated by your applications with [Error Tracking](error_tracking.md). - Create, toggle, and remove [Feature Flags](feature_flags.md). **(PREMIUM)** - [Trace](tracing.md) the performance and health of a deployed application. **(ULTIMATE)** -- Add a [button to the Monitoring dashboard](linking_to_an_external_dashboard.md) linking directly to your existing external dashboards. +- Change the [settings of the Monitoring Dashboard](dashboard_settings.md). diff --git a/doc/user/project/operations/linking_to_an_external_dashboard.md b/doc/user/project/operations/linking_to_an_external_dashboard.md index 8e1117de4c7..ff609a3720e 100644 --- a/doc/user/project/operations/linking_to_an_external_dashboard.md +++ b/doc/user/project/operations/linking_to_an_external_dashboard.md @@ -1,19 +1,5 @@ -# Linking to an external dashboard +--- +redirect_to: 'dashboard_settings.md' +--- -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/57171) in GitLab 12.0. - -You can add a button to the Monitoring dashboard linking directly to your existing external dashboards. - -## Enabling the external dashboard link - -1. Go to **Settings > Operations** and scroll to the section titled **External dashboard**. - -1. Fill in the URL to your external dashboard and click **Save changes**. - -  - -1. There should now be a button on your - [Monitoring dashboard](../../../ci/environments/index.md#monitoring-environments) which - will open the URL you entered in the above step. - -  +This document was moved to [another location](dashboard_settings.md). diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index a5fc5b00b53..bf9b58acf14 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -273,7 +273,7 @@ Sublime Text, Atom, Dreamweaver, Brackets, etc). ## Force HTTPS for GitLab Pages websites -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28857) in GitLab 10.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28857) in GitLab 10.7. To make your website's visitors even more secure, you can choose to force HTTPS for GitLab Pages. By doing so, all attempts to visit your @@ -287,6 +287,9 @@ To enable this setting: 1. Navigate to your project's **Settings > Pages**. 1. Tick the checkbox **Force HTTPS (requires valid certificates)**. +NOTE: **Note** +If you use CloudFlare CDN in front of GitLab Pages, make sure to set the SSL connection setting to `full` instead of `flexible`. For more details, see the [CloudFlare CDN directions](https://support.cloudflare.com/hc/en-us/articles/200170416-End-to-end-HTTPS-with-Cloudflare-Part-3-SSL-options#h_4e0d1a7c-eb71-4204-9e22-9d3ef9ef7fef). + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md index 8b180d438ef..4b4b430b663 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Pages integration with Let's Encrypt -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28996) in GitLab 12.1. For versions earlier than GitLab 12.1, see the [manual Let's Encrypt instructions](../lets_encrypt_for_gitlab_pages.md). +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28996) in GitLab 12.1. For versions earlier than GitLab 12.1, see the [manual Let's Encrypt instructions](../lets_encrypt_for_gitlab_pages.md). The GitLab Pages integration with Let's Encrypt (LE) allows you to use LE certificates for your Pages website with custom domains @@ -19,7 +19,7 @@ GitLab does it for you, out-of-the-box. open source Certificate Authority. CAUTION: **Caution:** -This feature covers only certificates for **custom domains**, not the wildcard certificate required to run [Pages daemon](../../../../administration/pages/index.md) **(CORE ONLY)**. Wildcard certificate generation is tracked in [this issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3342). +This feature covers only certificates for **custom domains**, not the wildcard certificate required to run [Pages daemon](../../../../administration/pages/index.md) **(CORE ONLY)**. Wildcard certificate generation is tracked in [this issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3342). ## Requirements diff --git a/doc/user/project/pages/getting_started/fork_sample_project.md b/doc/user/project/pages/getting_started/fork_sample_project.md index 00cea846f91..de9bd97b262 100644 --- a/doc/user/project/pages/getting_started/fork_sample_project.md +++ b/doc/user/project/pages/getting_started/fork_sample_project.md @@ -5,68 +5,52 @@ group: Release Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# New Pages website from a forked sample - -To get started with GitLab Pages from a sample website, the easiest -way to do it is by using one of the [bundled templates](pages_bundled_template.md). -If you don't find one that suits your needs, you can opt by -forking (copying) a [sample project from the most popular Static Site Generators](https://gitlab.com/pages). - -<table class="borderless-table center fixed-table middle width-80"> - <tr> - <td style="width: 30%"><img src="../img/icons/fork.png" alt="Fork" class="image-noshadow half-width"></td> - <td style="width: 10%"> - <strong> - <i class="fa fa-angle-double-right" aria-hidden="true"></i> - </strong> - </td> - <td style="width: 30%"><img src="../img/icons/terminal.png" alt="Deploy" class="image-noshadow half-width"></td> - <td style="width: 10%"> - <strong> - <i class="fa fa-angle-double-right" aria-hidden="true"></i> - </strong> - </td> - <td style="width: 30%"><img src="../img/icons/click.png" alt="Visit" class="image-noshadow half-width"></td> - </tr> - <tr> - <td><em>Fork an example project</em></td> - <td></td> - <td><em>Deploy your website</em></td> - <td></td> - <td><em>Visit your website's URL</em></td> - </tr> -</table> - -**<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video tutorial](https://www.youtube.com/watch?v=TWqh9MtT4Bg) with all the steps below.** - -1. [Fork](../../../../gitlab-basics/fork-project.md) a sample project from the [GitLab Pages examples](https://gitlab.com/pages) group. -1. From the left sidebar, navigate to your project's **CI/CD > Pipelines** - and click **Run pipeline** to trigger GitLab CI/CD to build and deploy your - site to the server. -1. Once the pipeline has finished successfully, find the link to visit your - website from your project's **Settings > Pages**. It can take approximately - 30 minutes to be deployed. - -You can also take some **optional** further steps: - -- _Remove the fork relationship._ The fork relationship is necessary to contribute back to the project you originally forked from. If you don't have any intentions to do so, you can remove it. To do so, navigate to your project's **Settings**, expand **Advanced settings**, and scroll down to **Remove fork relationship**: - -  - -- _Make it a user or group website._ To turn a **project website** forked - from the Pages group into a **user/group** website, you'll need to: - - Rename it to `namespace.gitlab.io`: go to your project's - **Settings > General** and expand **Advanced**. Scroll down to - **Change path** and change the path to `namespace.gitlab.io`. - - For example, consider the group `https://gitlab.com/gitlab-tests`: - `gitlab-tests` is the group's namespace, the repository path should be set - to `gitlab-tests.gitlab.io` (yes, weird like that), and the - resulting URL for your Pages website will be `https://gitlab-tests.gitlab.io`. +# Create a Pages website from a forked sample + +GitLab provides [sample projects for the most popular Static Site Generators](https://gitlab.com/pages). +You can fork one of the sample projects and run the CI/CD pipeline to generate a Pages website. + +Fork a sample project when you want to test GitLab Pages or start a new project that's already +configured to generate a Pages site. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video tutorial](https://www.youtube.com/watch?v=TWqh9MtT4Bg) of how this works. + +To fork a sample project and create a Pages website: + +1. View the sample projects by going to the [GitLab Pages examples](https://gitlab.com/pages) group. +1. Click the name of the project you want to [fork](../../../../gitlab-basics/fork-project.md). +1. In the top right, click the **Fork** button, and then choose a namespace to fork to. +1. Go to your project's **CI/CD > Pipelines** and click **Run pipeline**. + GitLab CI/CD builds and deploys your site. + +The site can take approximately 30 minutes to deploy. +When the pipeline is finished, go to **Settings > Pages** to find the link to your website from your project. + +For every change pushed to your repository, GitLab CI/CD runs a new pipeline +that immediately publishes your changes to the Pages site. + +You can take some **optional** further steps: + +- _Remove the fork relationship._ If you want to contribute to the project you forked from, + you can keep this relationship. Otherwise, go to your project's **Settings > General**, + expand **Advanced settings**, and scroll down to **Remove fork relationship**: + +  + +- _Change the URL to match your namespace._ If your Pages site is hosted on GitLab.com, + you can rename it to `<namespace>.gitlab.io`, where `<namespace>` is your GitLab namespace + (the one you chose when you forked the project). + + - Go to your project's **Settings > General** and expand **Advanced**. Scroll down to + **Change path** and change the path to `<namespace>.gitlab.io`. + + For example, if your project's URL is `gitlab.com/gitlab-tests/jekyll`, your namespace is + `gitlab-tests`. + + If you set the repository path to `gitlab-tests.gitlab.io`, + the resulting URL for your Pages website is `https://gitlab-tests.gitlab.io`.  - - Adjust your SSG's [base URL](../getting_started_part_one.md#urls-and-baseurls) from `"project-name"` to - `""`. This setting will be at a different place for each SSG, as each of them - have their own structure and file tree. Most likely, it will be in the SSG's - config file. + - Now go to your SSG's configuration file and change the [base URL](../getting_started_part_one.md#urls-and-baseurls) + from `"project-name"` to `""`. The project name setting varies by SSG and may not be in the config file. diff --git a/doc/user/project/pages/getting_started/new_or_existing_website.md b/doc/user/project/pages/getting_started/new_or_existing_website.md index 9a00b724753..5d7126ab22e 100644 --- a/doc/user/project/pages/getting_started/new_or_existing_website.md +++ b/doc/user/project/pages/getting_started/new_or_existing_website.md @@ -5,44 +5,45 @@ group: Release Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Start a new Pages website from scratch or deploy an existing website +# Create a Pages website by using a CI/CD template -If you already have a website and want to deploy it with GitLab Pages, -or, if you want to start a new site from scratch, you'll need to: +GitLab provides `.gitlab-ci.yml` templates for the most popular Static Site Generators (SSGs). +You can create your own `.gitlab-ci.yml` file from one of these templates, and run +the CI/CD pipeline to generate a Pages website. -- Create a new project in GitLab to hold your site content. -- Set up GitLab CI/CD to deploy your website to Pages. +Use a `.gitlab-ci.yml` template when you have an existing project that you want to add a Pages site to. -To do so, follow the steps below. +Your GitLab repository should contain files specific to an SSG, or plain HTML. +After you complete these steps, you may need to do additional +configuration for the Pages site to generate properly. -1. From your **Project**'s **[Dashboard](https://gitlab.com/dashboard/projects)**, - click **New project**, and name it according to the - [Pages domain names](../getting_started_part_one.md#gitlab-pages-default-domain-names). -1. Clone it to your local computer, add your website - files to your project, add, commit, and push to GitLab. - Alternatively, you can run `git init` in your local directory, - add the remote URL: - `git remote add origin git@gitlab.com:namespace/project-name.git`, - then add, commit, and push to GitLab. -1. From the your **Project**'s page, click **Set up CI/CD**: +1. In the left sidebar, click **Project overview**. +1. Click **Set up CI/CD**. -  +  -1. Choose one of the templates from the dropbox menu. - Pick up the template corresponding to the SSG you're using (or plain HTML). + If this button is not available, CI/CD is already configured for + your project. You may want to browse the `.gitlab-ci.yml` files + [in these projects instead](https://gitlab.com/pages). -  +1. From the **Apply a template** list, choose a template for the SSG you're using. + You can also choose plain HTML. - Note that, if you don't find a corresponding template, you can look into - [GitLab Pages group of sample projects](https://gitlab.com/pages), - you may find one among them that suits your needs, from which you - can copy `.gitlab-ci.yml`'s content and adjust for your case. - If you don't find it there either, [learn how to write a `.gitlab-ci.yml` +  + + If you don't find a corresponding template, you can view the + [GitLab Pages group of sample projects](https://gitlab.com/pages). + These projects contain `.gitlab-ci.yml` files that you can modify for your needs. + You can also [learn how to write your own `.gitlab-ci.yml` file for GitLab Pages](../getting_started_part_four.md). -Once you have both site files and `.gitlab-ci.yml` in your project's -root, GitLab CI/CD will build your site and deploy it with Pages. -Once the first build passes, you access your site by -navigating to your **Project**'s **Settings** > **Pages**, -where you'll find its default URL. It can take approximately 30 min to be -deployed. +1. Save and commit the `.gitlab-ci.yml` file. + +If everything is configured correctly, the site can take approximately 30 minutes to deploy. + +You can watch the pipeline run by going to **CI / CD > Pipelines**. +When the pipeline is finished, go to **Settings > Pages** to find the link to +your Pages website. + +For every change pushed to your repository, GitLab CI/CD runs a new pipeline +that immediately publishes your changes to the Pages site. diff --git a/doc/user/project/pages/getting_started/pages_bundled_template.md b/doc/user/project/pages/getting_started/pages_bundled_template.md index 745c50e8d65..cfa4e0910db 100644 --- a/doc/user/project/pages/getting_started/pages_bundled_template.md +++ b/doc/user/project/pages/getting_started/pages_bundled_template.md @@ -5,27 +5,30 @@ group: Release Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# New Pages website from a bundled template +# Create a Pages website from a new project template -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47857) in GitLab 11.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47857) in GitLab 11.8. -The simplest way to create a GitLab Pages site is to use one of the most -popular templates, which come already bundled with GitLab and are ready to go. +GitLab provides templates for the most popular Static Site Generators (SSGs). +You can create a new project from a template and run the CI/CD pipeline to generate a Pages website. + +Use a template when you want to test GitLab Pages or start a new project that's already +configured to generate a Pages site. 1. From the top navigation, click the **+** button and select **New project**. 1. Select **Create from Template**. -1. Choose one of the templates starting with **Pages**: +1. Next to one of the templates starting with **Pages**, click **Use template**. -  +  +1. Complete the form and click **Create project**. 1. From the left sidebar, navigate to your project's **CI/CD > Pipelines** and click **Run pipeline** to trigger GitLab CI/CD to build and deploy your - site to the server. -1. After the pipeline has finished successfully, wait approximately 30 minutes - for your website to be visible. After waiting 30 minutes, find the link to - visit your website from your project's **Settings > Pages**. If the link - leads to a 404 page, wait a few minutes and try again. - -Your website is then visible on your domain and you can modify your files -as you wish. For every modification pushed to your repository, GitLab CI/CD -will run a new pipeline to immediately publish your changes to the server. + site. + +The site can take approximately 30 minutes to deploy. +When the pipeline is finished, go to **Settings > Pages** to find the link to +your Pages website. + +For every change pushed to your repository, GitLab CI/CD runs a new pipeline +that immediately publishes your changes to the Pages site. diff --git a/doc/user/project/pages/getting_started_part_four.md b/doc/user/project/pages/getting_started_part_four.md index 4e95b5d5a69..8cf58280b88 100644 --- a/doc/user/project/pages/getting_started_part_four.md +++ b/doc/user/project/pages/getting_started_part_four.md @@ -6,137 +6,146 @@ group: Release Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Creating and Tweaking GitLab CI/CD for GitLab Pages - -To [get started with GitLab Pages](index.md#getting-started), you can -use one of the project templates, a `.gitlab-ci.yml` template, -or fork an existing example project. Therefore, you don't need to -understand _all_ the ins and odds of GitLab CI/CD to get your site -deployed. Still, there are cases where you want to write your own -script or tweak an existing one. This document guides you through -this process. - -This guide also provides a general overview and clear introduction -for **getting familiar with the `.gitlab-ci.yml` file and writing -one for the first time.** - -[GitLab CI/CD](../../../ci/README.md) serves -numerous purposes, to build, test, and deploy your app -from GitLab through -[Continuous Integration, Continuous Delivery, and Continuous Deployment](../../../ci/introduction/index.md#introduction-to-cicd-methodologies) -methods. You will need it to build your website with GitLab Pages, -and deploy it to the Pages server. - -To implement GitLab CI/CD, the first thing you need is a configuration -file called `.gitlab-ci.yml` placed at your website's root directory. - -What this file actually does is telling the -[GitLab Runner](https://docs.gitlab.com/runner/) to run scripts -as you would do from the command line. The Runner acts as your -terminal. GitLab CI/CD tells the Runner which commands to run. -Both are built-in in GitLab, and you don't need to set up -anything for them to work. - -Explaining [every detail of GitLab CI/CD](../../../ci/yaml/README.md) -and GitLab Runner is out of the scope of this guide, but we'll -need to understand just a few things to be able to write our own -`.gitlab-ci.yml` or tweak an existing one. It's a -[YAML](https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html) file, -with its own syntax. You can always check your CI syntax with -the [GitLab CI/CD Lint Tool](https://gitlab.com/ci/lint). - -## Practical example - -Let's consider you have a [Jekyll](https://jekyllrb.com/) site. -To build it locally, you would open your terminal, and run `jekyll build`. -Of course, before building it, you had to install Jekyll in your computer. -For that, you had to open your terminal and run `gem install jekyll`. -Right? GitLab CI/CD + GitLab Runner do the same thing. But you need to -write in the `.gitlab-ci.yml` the script you want to run so -GitLab Runner will do it for you. It looks more complicated than it -is. What you need to tell the Runner: - -```shell -gem install jekyll -jekyll build +# Create a GitLab Pages website from scratch + +This tutorial shows you how to create a Pages site from scratch. You will start with +a blank project and create your own CI file, which gives instruction to the +[GitLab Runner](https://docs.gitlab.com/runner/). When your CI/CD +[pipeline](../../../ci/pipelines/index.md) runs, the Pages site is created. + +This example uses the [Jekyll](https://jekyllrb.com/) Static Site Generator (SSG). +Other SSGs follow similar steps. You do not need to be familiar with Jekyll or SSGs +to complete this tutorial. + +## Prerequisites + +To follow along with this example, start with a blank project in GitLab. +Create three files in the root (top-level) directory. + +- `.gitlab-ci.yml` A YAML file that contains the commands you want to run. + For now, leave the file's contents blank. + +- `index.html` An HTML file you can populate with whatever HTML content you'd like, + for example: + + ```html + <html> + <head> + <title>Home</title> + </head> + <body> + <h1>Hello World!</h1> + </body> + </html> + ``` + +- [`Gemfile`](https://bundler.io/gemfile.html) A file that describes dependencies for Ruby programs. + Populate it with this content: + + ```ruby + source "https://rubygems.org" + + gem "jekyll" + ``` + +## Choose a Docker image + +In this example, the Runner uses a [Docker image](../../../ci/docker/using_docker_images.md) +to run scripts and deploy the site. + +This specific Ruby image is maintained on [DockerHub](https://hub.docker.com/_/ruby). + +Edit your `.gitlab-ci.yml` and add this text as the first line. + +```yaml +image: ruby:2.7 ``` -### Script +If your SSG needs [NodeJS](https://nodejs.org/) to build, you must specify an +image that contains NodeJS as part of its file system. For example, for a +[Hexo](https://gitlab.com/pages/hexo) site, you can use `image: node:12.17.0`. + +## Install Jekyll + +To run [Jekyll](https://jekyllrb.com/) locally, you would open your terminal and: -To transpose this script to YAML, it would be like this: +- Install [Bundler](https://bundler.io/) by running `gem install bundler`. +- Create `Gemfile.lock` by running `bundle install`. +- Install Jekyll by running `bundle exec jekyll build`. + +In the `.gitlab-ci.yml` file, this is written as: ```yaml script: - - gem install jekyll - - jekyll build + - gem install bundler + - bundle install + - bundle exec jekyll build ``` -### Job - -So far so good. Now, each `script`, in GitLab is organized by -a `job`, which is a bunch of scripts and settings you want to -apply to that specific task. +In addition, in the `.gitlab-ci.yml` file, each `script` is organized by a `job`. +A `job` includes the scripts and settings you want to apply to that specific +task. ```yaml job: script: - - gem install jekyll - - jekyll build + - gem install bundler + - bundle install + - bundle exec jekyll build ``` -For GitLab Pages, this `job` has a specific name, called `pages`, -which tells the Runner you want that task to deploy your website +For GitLab Pages, this `job` has a specific name, called `pages`. +This setting tells the Runner you want the job to deploy your website with GitLab Pages: ```yaml pages: script: - - gem install jekyll - - jekyll build + - gem install bundler + - bundle install + - bundle exec jekyll build ``` -### The `public` directory +## Specify the `public` directory for output + +Jekyll needs to know where to generate its output. +GitLab Pages only considers files in a directory called `public`. -We also need to tell Jekyll where do you want the website to build, -and GitLab Pages will only consider files in a directory called `public`. -To do that with Jekyll, we need to add a flag specifying the -[destination (`-d`)](https://jekyllrb.com/docs/usage/) of the -built website: `jekyll build -d public`. Of course, we need -to tell this to our Runner: +Jekyll uses destination (`-d`) to specify an output directory for the built website: ```yaml pages: script: - - gem install jekyll - - jekyll build -d public + - gem install bundler + - bundle install + - bundle exec jekyll build -d public ``` -### Artifacts +## Specify the `public` directory for artifacts -We also need to tell the Runner that this _job_ generates -_artifacts_, which is the site built by Jekyll. -Where are these artifacts stored? In the `public` directory: +Now that Jekyll has output the files to the `public` directory, +the Runner needs to know where to get them. The artifacts are stored +in the `public` directory: ```yaml pages: script: - - gem install jekyll - - jekyll build -d public + - gem install bundler + - bundle install + - bundle exec jekyll build -d public artifacts: paths: - public ``` -The script above would be enough to build your Jekyll -site with GitLab Pages. But, from Jekyll 3.4.0 on, its default -template originated by `jekyll new project` requires -[Bundler](https://bundler.io) to install Jekyll dependencies -and the default theme. To adjust our script to meet these new -requirements, we only need to install and build Jekyll with Bundler: +Paste this into `.gitlab-ci.yml` file, so it now looks like this: ```yaml +image: ruby:2.7 + pages: script: + - gem install bundler - bundle install - bundle exec jekyll build -d public artifacts: @@ -144,27 +153,35 @@ pages: - public ``` -That's it! A `.gitlab-ci.yml` with the content above would deploy -your Jekyll 3.4.0 site with GitLab Pages. This is the minimum -configuration for our example. On the steps below, we'll refine -the script by adding extra options to our GitLab CI/CD. +Now save and commit the `.gitlab-ci.yml` file. You can watch the pipeline run +by going to **CI / CD > Pipelines**. + +When it succeeds, go to **Settings > Pages** to view the URL where your site +is now available. + +If you want to do more advanced tasks, you can update your `.gitlab-ci.yml` file +with [any of the available settings](../../../ci/yaml/README.md). You can check +your CI syntax with the [GitLab CI/CD Lint Tool](https://gitlab.com/ci/lint). + +The following topics show other examples of other options you can add to your CI/CD file. -Artifacts will be automatically deleted once GitLab Pages got deployed. -You can preserve artifacts for limited time by specifying the expiry time. +## Deploy specific branches to a Pages site -### Image +You may want to deploy to a Pages site only from specific branches. -At this point, you probably ask yourself: "okay, but to install Jekyll -I need Ruby. Where is Ruby on that script?". The answer is simple: the -first thing GitLab Runner will look for in your `.gitlab-ci.yml` is a -[Docker](https://www.docker.com/) image specifying what do you need in -your container to run that script: +First, add a `workflow` section to force the pipeline to run only when changes are +pushed to branches: ```yaml image: ruby:2.7 +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' + pages: script: + - gem install bundler - bundle install - bundle exec jekyll build -d public artifacts: @@ -172,134 +189,122 @@ pages: - public ``` -In this case, you're telling the Runner to pull this image, which -contains Ruby 2.7 as part of its file system. When you don't specify -this image in your configuration, the Runner will use a default -image, which is Ruby 2.6. - -If your SSG needs [NodeJS](https://nodejs.org/) to build, you'll -need to specify which image you want to use, and this image should -contain NodeJS as part of its file system. E.g., for a -[Hexo](https://gitlab.com/pages/hexo) site, you can use `image: node:4.2.2`. - ->**Note:** -We're not trying to explain what a Docker image is, -we just need to introduce the concept with a minimum viable -explanation. To know more about Docker images, please visit -their website or take a look at a -[summarized explanation](http://paislee.io/how-to-automate-docker-deployments/) here. - -Let's go a little further. - -### Branching - -If you use GitLab as a version control platform, you will have your -branching strategy to work on your project. Meaning, you will have -other branches in your project, but you'll want only pushes to the -default branch (usually `master`) to be deployed to your website. -To do that, we need to add another line to our CI, telling the Runner -to only perform that _job_ called `pages` on the `master` branch `only`: +Then configure the pipeline to run the job for the master branch only. ```yaml -image: ruby:2.6 +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' pages: script: + - gem install bundler - bundle install - bundle exec jekyll build -d public artifacts: paths: - public - only: - - master + rules: + - if: '$CI_COMMIT_BRANCH == "master"' ``` -### Stages +## Specify a stage to deploy -Another interesting concept to keep in mind are build stages. -Your web app can pass through a lot of tests and other tasks -until it's deployed to staging or production environments. -There are three default stages on GitLab CI/CD: build, test, -and deploy. To specify which stage your _job_ is running, -simply add another line to your CI: +There are three default stages for GitLab CI/CD: build, test, +and deploy. + +If you want to test your script and check the built site before deploying +to production, you can run the test exactly as it will run when you +push to `master`. + +To specify a stage for your job to run in, +add a `stage` line to your CI file: ```yaml -image: ruby:2.6 +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' pages: stage: deploy script: + - gem install bundler - bundle install - bundle exec jekyll build -d public artifacts: paths: - public - only: - - master + rules: + - if: '$CI_COMMIT_BRANCH == "master"' ``` -You might ask yourself: "why should I bother with stages -at all?" Well, let's say you want to be able to test your -script and check the built site before deploying your site -to production. You want to run the test exactly as your -script will do when you push to `master`. It's simple, -let's add another task (_job_) to our CI, telling it to -test every push to other branches, `except` the `master` branch: +Now add another job to the CI file, telling it to +test every push to every branch **except** the `master` branch: ```yaml -image: ruby:2.6 +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' pages: stage: deploy script: + - gem install bundler - bundle install - bundle exec jekyll build -d public artifacts: paths: - public - only: - - master + rules: + - if: '$CI_COMMIT_BRANCH == "master"' test: stage: test script: + - gem install bundler - bundle install - bundle exec jekyll build -d test artifacts: paths: - test - except: - - master + rules: + - if: '$CI_COMMIT_BRANCH != "master"' ``` -The `test` job is running on the stage `test`, Jekyll -will build the site in a directory called `test`, and -this job will affect all the branches except `master`. - -The best benefit of applying _stages_ to different -_jobs_ is that every job in the same stage builds in -parallel. So, if your web app needs more than one test -before being deployed, you can run all your test at the -same time, it's not necessary to wait one test to finish -to run the other. Of course, this is just a brief -introduction of GitLab CI/CD and GitLab Runner, which are -tools much more powerful than that. This is what you -need to be able to create and tweak your builds for -your GitLab Pages site. - -### Before Script - -To avoid running the same script multiple times across -your _jobs_, you can add the parameter `before_script`, -in which you specify which commands you want to run for -every single _job_. In our example, notice that we run -`bundle install` for both jobs, `pages` and `test`. -We don't need to repeat it: +When the `test` job runs in the `test` stage, Jekyll +builds the site in a directory called `test`. The job affects +all branches except `master`. + +When you apply stages to different jobs, every job in the same +stage builds in parallel. If your web application needs more than +one test before being deployed, you can run all your tests at the +same time. + +## Remove duplicate commands + +To avoid duplicating the same scripts in every job, you can add them +to a `before_script` section. + +In the example, `gem install bundler` and `bundle install` were running +for both jobs, `pages` and `test`. + +Move these commands to a `before_script` section: ```yaml -image: ruby:2.6 +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' before_script: + - gem install bundler - bundle install pages: @@ -309,8 +314,8 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: '$CI_COMMIT_BRANCH == "master"' test: stage: test @@ -319,26 +324,31 @@ test: artifacts: paths: - test - except: - - master + rules: + - if: '$CI_COMMIT_BRANCH != "master"' ``` -### Caching Dependencies +## Build faster with cached dependencies + +To build faster, you can cache the installation files for your +project's dependencies by using the `cache` parameter. -If you want to cache the installation files for your -projects dependencies, for building faster, you can -use the parameter `cache`. For this example, we'll -cache Jekyll dependencies in a `vendor` directory -when we run `bundle install`: +This example caches Jekyll dependencies in a `vendor` directory +when you run `bundle install`: ```yaml -image: ruby:2.6 +image: ruby:2.7 + +workflow: + rules: + - if: '$CI_COMMIT_BRANCH' cache: paths: - vendor/ before_script: + - gem install bundler - bundle install --path vendor pages: @@ -348,8 +358,8 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: '$CI_COMMIT_BRANCH == "master"' test: stage: test @@ -358,40 +368,35 @@ test: artifacts: paths: - test - except: - - master + rules: + - if: '$CI_COMMIT_BRANCH != "master"' ``` -For this specific case, we need to exclude `/vendor` -from Jekyll `_config.yml` file, otherwise Jekyll will -understand it as a regular directory to build -together with the site: +In this case, you need to exclude the `/vendor` +directory from the list of folders Jekyll builds. Otherwise, Jekyll +will try to build the directory contents along with the site. + +In the root directory, create a file called `_config.yml` +and add this content: ```yaml exclude: - vendor ``` -There we go! Now our GitLab CI/CD not only builds our website, -but also **continuously test** pushes to feature-branches, +Now GitLab CI/CD not only builds the website, +but also pushes with **continuous tests** to feature-branches, **caches** dependencies installed with Bundler, and -**continuously deploy** every push to the `master` branch. +**continuously deploys** every push to the `master` branch. -## Advanced GitLab CI for GitLab Pages +## Related topics -What you can do with GitLab CI/CD is pretty much up to your -creativity. Once you get used to it, you start creating -awesome scripts that automate most of tasks you'd do -manually in the past. Read through the -[documentation of GitLab CI/CD](../../../ci/yaml/README.md) -to understand how to go even further on your scripts. +For more information, see the following blog posts. -- On this blog post, understand the concept of - [using GitLab CI/CD `environments` to deploy your +- [Use GitLab CI/CD `environments` to deploy your web app to staging and production](https://about.gitlab.com/blog/2016/08/26/ci-deployment-and-environments/). -- On this post, learn [how to run jobs sequentially, - in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2016/07/29/the-basics-of-gitlab-ci/) -- On this blog post, we go through the process of - [pulling specific directories from different projects](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/) - to deploy this website you're looking at, <https://docs.gitlab.com>. -- On this blog post, we teach you [how to use GitLab Pages to produce a code coverage report](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/). +- Learn [how to run jobs sequentially, + in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2016/07/29/the-basics-of-gitlab-ci/). +- Learn [how to pull specific directories from different projects](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/) + to deploy this website, <https://docs.gitlab.com>. +- Learn [how to use GitLab Pages to produce a code coverage report](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/). diff --git a/doc/user/project/pages/img/change_path_v12_10.png b/doc/user/project/pages/img/change_path_v12_10.png Binary files differindex 7ca09bd21a3..9b5c17f1dda 100644 --- a/doc/user/project/pages/img/change_path_v12_10.png +++ b/doc/user/project/pages/img/change_path_v12_10.png diff --git a/doc/user/project/pages/img/choose_ci_template.png b/doc/user/project/pages/img/choose_ci_template.png Binary files differdeleted file mode 100644 index 0697542abc8..00000000000 --- a/doc/user/project/pages/img/choose_ci_template.png +++ /dev/null diff --git a/doc/user/project/pages/img/choose_ci_template_v13_1.png b/doc/user/project/pages/img/choose_ci_template_v13_1.png Binary files differnew file mode 100644 index 00000000000..84dd9fe2e0f --- /dev/null +++ b/doc/user/project/pages/img/choose_ci_template_v13_1.png diff --git a/doc/user/project/pages/img/pages_project_templates_v11_8.png b/doc/user/project/pages/img/pages_project_templates_v11_8.png Binary files differdeleted file mode 100644 index 61cae88b5a8..00000000000 --- a/doc/user/project/pages/img/pages_project_templates_v11_8.png +++ /dev/null diff --git a/doc/user/project/pages/img/pages_project_templates_v13_1.png b/doc/user/project/pages/img/pages_project_templates_v13_1.png Binary files differnew file mode 100644 index 00000000000..3f6d1ecd786 --- /dev/null +++ b/doc/user/project/pages/img/pages_project_templates_v13_1.png diff --git a/doc/user/project/pages/img/remove_fork_relationship.png b/doc/user/project/pages/img/remove_fork_relationship.png Binary files differdeleted file mode 100644 index 67c45491f08..00000000000 --- a/doc/user/project/pages/img/remove_fork_relationship.png +++ /dev/null diff --git a/doc/user/project/pages/img/remove_fork_relationship_v13_1.png b/doc/user/project/pages/img/remove_fork_relationship_v13_1.png Binary files differnew file mode 100644 index 00000000000..1bc7bcd253b --- /dev/null +++ b/doc/user/project/pages/img/remove_fork_relationship_v13_1.png diff --git a/doc/user/project/pages/img/setup_ci.png b/doc/user/project/pages/img/setup_ci.png Binary files differdeleted file mode 100644 index 214c1cc668f..00000000000 --- a/doc/user/project/pages/img/setup_ci.png +++ /dev/null diff --git a/doc/user/project/pages/img/setup_ci_v13_1.png b/doc/user/project/pages/img/setup_ci_v13_1.png Binary files differnew file mode 100644 index 00000000000..2cf1c05d6d8 --- /dev/null +++ b/doc/user/project/pages/img/setup_ci_v13_1.png diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index e81c9699153..5861282ca6f 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -1,7 +1,5 @@ --- description: 'Learn how to use GitLab Pages to deploy a static website at no additional cost.' -last_updated: 2019-06-04 -type: index, reference stage: Release group: Release Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers @@ -11,46 +9,76 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80) in GitLab Enterprise Edition 8.3. > - Custom CNAMEs with TLS support were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173) in GitLab Enterprise Edition 8.5. -> - [Ported](https://gitlab.com/gitlab-org/gitlab-foss/issues/14605) to GitLab Community Edition in GitLab 8.17. -> - Support for subgroup project's websites was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/30548) in GitLab 11.8. -> - Bundled project templates were [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/47857) in GitLab 11.8. +> - [Ported](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/14605) to GitLab Community Edition in GitLab 8.17. +> - Support for subgroup project's websites was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30548) in GitLab 11.8. +> - Bundled project templates were [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47857) in GitLab 11.8. -**GitLab Pages is a feature that allows you to publish static websites -directly from a repository in GitLab.** - -You can use it either for personal or business websites, such as -portfolios, documentation, manifestos, and business presentations. -You can also attribute any license to your content. - -<img src="img/pages_workflow_v12_5.png" alt="Pages websites workflow" class="image-noshadow"> - -Pages is available for free for all GitLab.com users as well as for self-managed -instances (GitLab Core, Starter, Premium, and Ultimate). - -## Overview +With GitLab Pages, you can publish static websites +directly from a repository in GitLab. <div class="row"> <div class="col-md-9"> <p style="margin-top: 18px;"> -To publish a website with Pages, you can use any Static Site Generator (SSG), -such as Gatsby, Jekyll, Hugo, Middleman, Harp, Hexo, and Brunch, just to name a few. You can also -publish any website written directly in plain HTML, CSS, and JavaScript.</p> -<p>Pages does <strong>not</strong> support dynamic server-side processing, for instance, as <code>.php</code> and <code>.asp</code> requires. See this article to learn more about -<a href="https://about.gitlab.com/blog/2016/06/03/ssg-overview-gitlab-pages-part-1-dynamic-x-static/">static websites vs dynamic websites</a>.</p> +<ul> +<li>Use for any personal or business website.</li> +<li>Use any Static Site Generator (SSG) or plain HTML.</li> +<li>Create websites for your projects, groups, or user account.</li> +<li>Host your site on your own GitLab instance or on GitLab.com for free.</li> +<li>Connect your custom domains and TLS certificates.</li> +<li>Attribute any license to your content.</li> +</ul> +</p> </div> <div class="col-md-3"><img src="img/ssgs_pages.png" alt="Examples of SSGs supported by Pages" class="image-noshadow middle display-block"></div> </div> -### How it works +To publish a website with Pages, you can use any SSG, +like Gatsby, Jekyll, Hugo, Middleman, Harp, Hexo, and Brunch, just to name a few. You can also +publish any website written directly in plain HTML, CSS, and JavaScript. -To use GitLab Pages, first you need to create a project in GitLab to upload your website's -files to. These projects can be either public, internal, or private, at your own choice. +Pages does **not** support dynamic server-side processing, for instance, as `.php` and `.asp` requires. +Learn more about +[static websites compared to dynamic websites](https://about.gitlab.com/blog/2016/06/03/ssg-overview-gitlab-pages-part-1-dynamic-x-static/). + +## Getting started + +To create a GitLab Pages website: + +| Document | Description | +| -------- | ----------- | +| [Use a `.gitlab-ci.yml` template](getting_started/new_or_existing_website.md) | Add a Pages site to an existing project. Use a pre-populated CI template file. | +| [Create a `gitlab-ci.yml` file from scratch](getting_started_part_four.md) | Add a Pages site to an existing project. Learn how to create and configure your own CI file. | +| [Use a new project template](getting_started/pages_bundled_template.md) | Create a new project with Pages already configured by using a new project template. | +| [Fork a sample project](getting_started/fork_sample_project.md) | Create a new project with Pages already configured by forking a sample project. | + +To update a GitLab Pages website: + +| Document | Description | +| -------- | ----------- | +| [GitLab Pages domain names, URLs, and base URLs](getting_started_part_one.md) | Learn about GitLab Pages default domains. | +| [Explore GitLab Pages](introduction.md) | Requirements, technical aspects, specific GitLab CI/CD configuration options, Access Control, custom 404 pages, limitations, FAQ. | +| [Custom domains and SSL/TLS Certificates](custom_domains_ssl_tls_certification/index.md) | Custom domains and subdomains, DNS records, and SSL/TLS certificates. | +| [Let's Encrypt integration](custom_domains_ssl_tls_certification/lets_encrypt_integration.md) | Secure your Pages sites with Let's Encrypt certificates, which are automatically obtained and renewed by GitLab. | +| [CloudFlare certificates](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/) | Secure your Pages site with CloudFlare certificates. | + +Learn more and see examples: + +| Document | Description | +| -------- | ----------- | +| [Static vs dynamic websites](https://about.gitlab.com/blog/2016/06/03/ssg-overview-gitlab-pages-part-1-dynamic-x-static/) | Static versus dynamic site overview. | +| [Modern static site generators](https://about.gitlab.com/blog/2016/06/10/ssg-overview-gitlab-pages-part-2/) | SSG overview. | +| [Build any SSG site with GitLab Pages](https://about.gitlab.com/blog/2016/06/17/ssg-overview-gitlab-pages-part-3-examples-ci/) | Use SSGs for GitLab Pages. | -GitLab will always deploy your website from a very specific folder called `public` in your -repository. Note that when you create a new project in GitLab, a [repository](../repository/index.md) +## How it works + +To use GitLab Pages, you must create a project in GitLab to upload your website's +files to. These projects can be either public, internal, or private. + +GitLab always deploys your website from a very specific folder called `public` in your +repository. When you create a new project in GitLab, a [repository](../repository/index.md) becomes available automatically. -To deploy your site, GitLab will use its built-in tool called [GitLab CI/CD](../../../ci/README.md), +To deploy your site, GitLab uses its built-in tool called [GitLab CI/CD](../../../ci/README.md) to build your site and publish it to the GitLab Pages server. The sequence of scripts that GitLab CI/CD runs to accomplish this task is created from a file named `.gitlab-ci.yml`, which you can [create and modify](getting_started_part_four.md) at will. A specific `job` called `pages` in the configuration file will make GitLab aware that you are deploying a GitLab Pages website. @@ -59,59 +87,29 @@ You can either use GitLab's [default domain for GitLab Pages websites](getting_s `*.gitlab.io`, or your own domain (`example.com`). In that case, you'll need admin access to your domain's registrar (or control panel) to set it up with Pages. -### Getting started - -To get started with GitLab Pages, you can either: - -- [Use a bundled website template ready to go](getting_started/pages_bundled_template.md). -- [Copy an existing sample](getting_started/fork_sample_project.md). -- [Create a website from scratch or deploy an existing one](getting_started/new_or_existing_website.md). +The following diagrams show the workflows you might follow to get started with Pages. <img src="img/new_project_for_pages_v12_5.png" alt="New projects for GitLab Pages" class="image-noshadow"> -Optional features: +## Access to your Pages site -- Use a [custom domain or subdomain](custom_domains_ssl_tls_certification/index.md#set-up-pages-with-a-custom-domain). -- Add an [SSL/TLS certificate to secure your site under the HTTPS protocol](custom_domains_ssl_tls_certification/index.md#adding-an-ssltls-certificate-to-pages). - -Note that, if you're using GitLab Pages default domain (`.gitlab.io`), +If you're using GitLab Pages default domain (`.gitlab.io`), your website will be automatically secure and available under HTTPS. If you're using your own custom domain, you can optionally secure it with SSL/TLS certificates. -## Availability - If you're using GitLab.com, your website will be publicly available to the internet. - To restrict access to your website, enable [GitLab Pages Access Control](pages_access_control.md). -If you're using self-managed instances (Core, Starter, Premium, or Ultimate), +If you're using a self-managed instance (Core, Starter, Premium, or Ultimate), your websites will be published on your own server, according to the [Pages admin settings](../../../administration/pages/index.md) chosen by your sysadmin, -who can opt for making them public or internal to your server. +who can make them public or internal. -## Explore GitLab Pages +## Pages examples -To learn more about configuration options for GitLab Pages, read the following: - -| Document | Description | -| --- | --- | -| [GitLab Pages domain names, URLs, and baseurls](getting_started_part_one.md) | Understand how GitLab Pages default domains work. | -| [GitLab CI/CD for GitLab Pages](getting_started_part_four.md) | Understand how to create your own `.gitlab-ci.yml` for your site. | -| [Exploring GitLab Pages](introduction.md) | Requirements, technical aspects, specific GitLab CI/CD's configuration options, Access Control, custom 404 pages, limitations, FAQ. | -|---+---| -| [Custom domains and SSL/TLS Certificates](custom_domains_ssl_tls_certification/index.md) | How to add custom domains and subdomains to your website, configure DNS records and SSL/TLS certificates. | -| [Let's Encrypt integration](custom_domains_ssl_tls_certification/lets_encrypt_integration.md) | Secure your Pages sites with Let's Encrypt certificates automatically obtained and renewed by GitLab. | -| [CloudFlare certificates](https://about.gitlab.com/blog/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/) | Secure your Pages site with CloudFlare certificates. | -|---+---| -| [Static vs dynamic websites](https://about.gitlab.com/blog/2016/06/03/ssg-overview-gitlab-pages-part-1-dynamic-x-static/) | A conceptual overview on static versus dynamic sites. | -| [Modern static site generators](https://about.gitlab.com/blog/2016/06/10/ssg-overview-gitlab-pages-part-2/) | A conceptual overview on SSGs. | -| [Build any SSG site with GitLab Pages](https://about.gitlab.com/blog/2016/06/17/ssg-overview-gitlab-pages-part-3-examples-ci/) | An overview on using SSGs for GitLab Pages. | - -## Advanced use - -There are quite some great examples of GitLab Pages websites built for some -specific reasons. These examples can teach you some advanced techniques +There are some great examples of GitLab Pages websites built for +specific reasons. These examples can teach you advanced techniques to use and adapt to your own needs: - [Posting to your GitLab Pages blog from iOS](https://about.gitlab.com/blog/2016/08/19/posting-to-your-gitlab-pages-blog-from-ios/). @@ -120,14 +118,9 @@ to use and adapt to your own needs: - [Building a new GitLab docs site with Nanoc, GitLab CI, and GitLab Pages](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/). - [Publish code coverage reports with GitLab Pages](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/). -## Admin GitLab Pages for self-managed instances - -Enable and configure GitLab Pages on your own instance (GitLab Community Edition and Enterprise Editions) with -the [admin guide](../../../administration/pages/index.md). - -**<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video tutorial](https://www.youtube.com/watch?v=dD8c7WNcc6s) for getting started with GitLab Pages admin!** +## Administer GitLab Pages for self-managed instances -## More information about GitLab Pages +If you are running a self-managed instance of GitLab (GitLab Community Edition and Enterprise Editions), +[follow the administration steps](../../../administration/pages/index.md) to configure Pages. -- Announcement (2016-12-24): ["We're bringing GitLab Pages to CE"](https://about.gitlab.com/releases/2016/12/24/were-bringing-gitlab-pages-to-community-edition/) -- Announcement (2017-03-06): ["We are changing the IP of GitLab Pages on GitLab.com"](https://about.gitlab.com/releases/2017/03/06/we-are-changing-the-ip-of-gitlab-pages-on-gitlab-com/) +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a [video tutorial](https://www.youtube.com/watch?v=dD8c7WNcc6s) about how to get started with GitLab Pages administration. diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index e36dfd89ab3..614a0d0dd19 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -25,7 +25,7 @@ In brief, this is what you need to upload your website in GitLab Pages: 1. Domain of the instance: domain name that is used for GitLab Pages (ask your administrator). 1. GitLab CI/CD: a `.gitlab-ci.yml` file with a specific job named [`pages`](../../../ci/yaml/README.md#pages) in the root directory of your repository. -1. A directory called `public` in your site's repo containing the content +1. A directory called `public` in your site's repository containing the content to be published. 1. GitLab Runner enabled for the project. @@ -87,7 +87,7 @@ will be deleted. When using Pages under the general domain of a GitLab instance (`*.example.io`), you _cannot_ use HTTPS with sub-subdomains. That means that if your -username/groupname contains a dot, for example `foo.bar`, the domain +username or group name contains a dot, for example `foo.bar`, the domain `https://foo.bar.example.io` will _not_ work. This is a limitation of the [HTTP Over TLS protocol](https://tools.ietf.org/html/rfc2818#section-3.1). HTTP pages will continue to work provided you don't redirect HTTP to HTTPS. @@ -217,7 +217,7 @@ needing to compress files on-demand. ### Resolving ambiguous URLs -> [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/issues/95) in GitLab 11.8 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/95) in GitLab 11.8 GitLab Pages makes assumptions about which files to serve when receiving a request for a URL that does not include an extension. diff --git a/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md b/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md index 33181828fb2..d86704eb703 100644 --- a/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md +++ b/doc/user/project/pages/lets_encrypt_for_gitlab_pages.md @@ -105,7 +105,7 @@ operating systems the steps might be slightly different. Follow the therefore, it needs to be part of the website content under the repo's [`public`](index.md#how-it-works) folder. -1. Add, commit, and push the file into your repo in GitLab. Once the pipeline +1. Add, commit, and push the file into your repository in GitLab. Once the pipeline passes, press **Enter** on your terminal to continue issuing your certificate. CertBot will then prompt you with the following message: diff --git a/doc/user/project/pages/pages_access_control.md b/doc/user/project/pages/pages_access_control.md index 7fe4c4c051d..6fcad0a5357 100644 --- a/doc/user/project/pages/pages_access_control.md +++ b/doc/user/project/pages/pages_access_control.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Pages Access Control -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/33422) in GitLab 11.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/33422) in GitLab 11.5. > - Available on GitLab.com in GitLab 12.4. You can enable Pages access control on your project, so that only diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index e2ee0dd47fe..d0baf57f169 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -2,7 +2,7 @@ type: reference, howto --- -# Protected Branches +# Protected branches [Permissions](../permissions.md) in GitLab are fundamentally defined around the idea of having read or write permission to the repository and branches. To impose @@ -114,7 +114,7 @@ all matching branches: ## Creating a protected branch -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/53361) in GitLab 11.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53361) in GitLab 11.9. When a protected branch or wildcard protected branches are set to [**No one** is **Allowed to push**](#using-the-allowed-to-merge-and-allowed-to-push-settings), @@ -134,7 +134,7 @@ To create a new branch through the user interface: ## Deleting a protected branch -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/21393) in GitLab 9.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/21393) in GitLab 9.3. From time to time, it may be required to delete or clean up branches that are protected. @@ -155,7 +155,7 @@ command line or a Git client application. ## Protected Branches approval by Code Owners **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13251) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13251) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.4. It is possible to require at least one approval by a [Code Owner](code_owners.md) to a file changed by the @@ -194,11 +194,11 @@ for details about the pipelines security model. **11.9** -- [Allow protected branches to be created](https://gitlab.com/gitlab-org/gitlab-foss/issues/53361) by Developers (and users with higher permission levels) through the API and the user interface. +- [Allow protected branches to be created](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53361) by Developers (and users with higher permission levels) through the API and the user interface. **9.2** -- Allow deletion of protected branches via the web interface ([issue #21393](https://gitlab.com/gitlab-org/gitlab-foss/issues/21393)). +- Allow deletion of protected branches via the web interface ([issue #21393](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/21393)). **8.11** diff --git a/doc/user/project/protected_tags.md b/doc/user/project/protected_tags.md index b134d283ba9..e80b8098bec 100644 --- a/doc/user/project/protected_tags.md +++ b/doc/user/project/protected_tags.md @@ -2,13 +2,13 @@ type: reference, howto --- -# Protected Tags +# Protected tags > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10356) in GitLab 9.1. -Protected Tags allow control over who has permission to create tags as well as preventing accidental update or deletion once created. Each rule allows you to match either an individual tag name, or use wildcards to control multiple tags at once. +Protected tags allow control over who has permission to create tags as well as preventing accidental update or deletion once created. Each rule allows you to match either an individual tag name, or use wildcards to control multiple tags at once. -This feature evolved out of [Protected Branches](protected_branches.md) +This feature evolved out of [protected branches](protected_branches.md) ## Overview diff --git a/doc/user/project/push_options.md b/doc/user/project/push_options.md index eab88d59867..ca658c06647 100644 --- a/doc/user/project/push_options.md +++ b/doc/user/project/push_options.md @@ -36,7 +36,7 @@ You can use push options to skip a CI/CD pipeline, or pass environment variables | Push option | Description | Introduced in version | | ------------------------------ | ------------------------------------------------------------------------------------------- |---------------------- | | `ci.skip` | Do not create a CI pipeline for the latest push. | [11.7](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/15643) | -| `ci.variable="<name>=<value>"` | Provide [environment variables](../../ci/variables/README.md) to be used in a CI pipeline, if one is created due to the push. | [12.6](https://gitlab.com/gitlab-org/gitlab/issues/27983) | +| `ci.variable="<name>=<value>"` | Provide [environment variables](../../ci/variables/README.md) to be used in a CI pipeline, if one is created due to the push. | [12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/27983) | An example of using `ci.skip`: @@ -60,9 +60,9 @@ time as pushing changes: | `merge_request.create` | Create a new merge request for the pushed branch. | [11.10](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26752) | | `merge_request.target=<branch_name>` | Set the target of the merge request to a particular branch. | [11.10](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26752) | | `merge_request.merge_when_pipeline_succeeds` | Set the merge request to [merge when its pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md). | [11.10](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26752) | -| `merge_request.remove_source_branch` | Set the merge request to remove the source branch when it's merged. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/issues/64320) | -| `merge_request.title="<title>"` | Set the title of the merge request. Ex: `git push -o merge_request.title="The title I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/issues/64320) | -| `merge_request.description="<description>"` | Set the description of the merge request. Ex: `git push -o merge_request.description="The description I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/issues/64320) | +| `merge_request.remove_source_branch` | Set the merge request to remove the source branch when it's merged. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | +| `merge_request.title="<title>"` | Set the title of the merge request. Ex: `git push -o merge_request.title="The title I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | +| `merge_request.description="<description>"` | Set the description of the merge request. Ex: `git push -o merge_request.description="The description I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | | `merge_request.label="<label>"` | Add labels to the merge request. If the label does not exist, it will be created. For example, for two labels: `git push -o merge_request.label="label1" -o merge_request.label="label2"`. | [12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31831) | | `merge_request.unlabel="<label>"` | Remove labels from the merge request. For example, for two labels: `git push -o merge_request.unlabel="label1" -o merge_request.unlabel="label2"`. | [12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31831) | diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index e2d0b616e4b..a3df173bd9d 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -1,5 +1,8 @@ --- type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab Quick Actions @@ -10,7 +13,8 @@ You can enter these commands while creating a new issue or merge request, or in comments of issues, epics, merge requests, and commits. Each command should be on a separate line in order to be properly detected and executed. -> From [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26672), once an action is executed, an alert is displayed when a quick action is successfully applied. +> From [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26672), once an +> action is executed, an alert appears when a quick action is successfully applied. ## Quick Actions for issues, merge requests and epics @@ -18,63 +22,66 @@ The following quick actions are applicable to descriptions, discussions and thre - Issues - Merge requests -- Epics **(ULTIMATE)** - -| Command | Issue | Merge request | Epic | Action | -|:--------------------------------------|:------|:--------------|:-----|:------ | -| `/tableflip <comment>` | ✓ | ✓ | ✓ | Append the comment with `(╯°□°)╯︵ ┻━┻` | -| `/shrug <comment>` | ✓ | ✓ | ✓ | Append the comment with `¯\＿(ツ)＿/¯` | -| `/todo` | ✓ | ✓ | ✓ | Add a To Do | -| `/done` | ✓ | ✓ | ✓ | Mark To Do as done | -| `/subscribe` | ✓ | ✓ | ✓ | Subscribe | -| `/unsubscribe` | ✓ | ✓ | ✓ | Unsubscribe | -| `/close` | ✓ | ✓ | ✓ | Close | -| `/reopen` | ✓ | ✓ | ✓ | Reopen | -| `/title <new title>` | ✓ | ✓ | ✓ | Change title | -| `/award :emoji:` | ✓ | ✓ | ✓ | Toggle emoji award | -| `/assign me` | ✓ | ✓ | | Assign yourself | -| `/assign @user` | ✓ | ✓ | | Assign one user | -| `/assign @user1 @user2` | ✓ | ✓ | | Assign multiple users **(STARTER)** | -| `/reassign @user1 @user2` | ✓ | ✓ | | Change assignee **(STARTER)** | -| `/unassign` | ✓ | ✓ | | Remove current assignee | -| `/unassign @user1 @user2` | ✓ | ✓ | | Remove assignee(s) **(STARTER)** | -| `/milestone %milestone` | ✓ | ✓ | | Set milestone | -| `/remove_milestone` | ✓ | ✓ | | Remove milestone | -| `/label ~label1 ~label2` | ✓ | ✓ | ✓ | Add label(s). Label names can also start without `~` but mixed syntax is not supported | -| `/relabel ~label1 ~label2` | ✓ | ✓ | ✓ | Replace existing label(s) with those specified | -| `/unlabel ~label1 ~label2` or `/remove_label ~label1 ~label2` | ✓ | ✓ | ✓ | Remove all or specific label(s) | -| `/copy_metadata <#issue>` | ✓ | ✓ | | Copy labels and milestone from another issue in the project | -| `/copy_metadata <!merge_request>` | ✓ | ✓ | | Copy labels and milestone from another merge request in the project | -| `/estimate <<W>w <DD>d <hh>h <mm>m>` | ✓ | ✓ | | Set time estimate. For example, `/estimate 1w 3d 2h 14m` | -| `/remove_estimate` | ✓ | ✓ | | Remove time estimate | -| `/spend <time(<h>h <mm>m)> <date(<YYYY-MM-DD>)>` | ✓ | ✓ | | Add spent time; optionally specify the date that time was spent on. For example, `/spend time(1h 30m)` or `/spend time(1h 30m) date(2018-08-26)` | -| `/spend <time(-<h>h <mm>m)> <date(<YYYY-MM-DD>)>` | ✓ | ✓ | | Subtract spent time; optionally specify the date that time was spent on. For example, `/spend time(-1h 30m)` or `/spend time(-1h 30m) date(2018-08-26)` | -| `/remove_time_spent` | ✓ | ✓ | | Remove time spent | -| `/lock` | ✓ | ✓ | | Lock the thread | -| `/unlock` | ✓ | ✓ | | Unlock the thread | -| `/due <date>` | ✓ | | | Set due date. Examples of valid `<date>` include `in 2 days`, `this Friday` and `December 31st` | -| `/remove_due_date` | ✓ | | | Remove due date | -| `/weight <value>` | ✓ | | | Set weight. Valid options for `<value>` include `0`, `1`, `2`, etc **(STARTER)** | -| `/clear_weight` | ✓ | | | Clear weight **(STARTER)** | -| `/epic <epic>` | ✓ | | | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. **(ULTIMATE)** | -| `/remove_epic` | ✓ | | | Remove from epic **(ULTIMATE)** | -| `/promote` | ✓ | | | Promote issue to epic **(ULTIMATE)** | -| `/confidential` | ✓ | | | Make confidential | -| `/duplicate <#issue>` | ✓ | | | Mark this issue as a duplicate of another issue and relate them for **(STARTER)** | -| `/create_merge_request <branch name>` | ✓ | | | Create a new merge request starting from the current issue | -| `/relate #issue1 #issue2` | ✓ | | | Mark issues as related **(STARTER)** | -| `/move <path/to/project>` | ✓ | | | Move this issue to another project | -| `/zoom <Zoom URL>` | ✓ | | | Add Zoom meeting to this issue. ([Introduced in GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609)) | -| `/remove_zoom` | ✓ | | | Remove Zoom meeting from this issue. ([Introduced in GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609)) | -| `/target_branch <local branch name>` | | ✓ | | Set target branch | -| `/wip` | | ✓ | | Toggle the Work In Progress status | -| `/approve` | | ✓ | | Approve the merge request **(STARTER)** | -| `/submit_review` | | ✓ | | Submit a pending review. ([Introduced in GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/issues/8041)) **(PREMIUM)** | -| `/merge` | | ✓ | | Merge changes. Depending on the project setting, this may be [when the pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md), adding to a [Merge Train](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md), etc). | -| `/child_epic <epic>` | | | ✓ | Add child epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. ([Introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/issues/7330)) **(ULTIMATE)** | -| `/remove_child_epic <epic>` | | | ✓ | Remove child epic from `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. ([Introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/issues/7330)) **(ULTIMATE)** | -| `/parent_epic <epic>` | | | ✓ | Set parent epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/issues/10556)) **(ULTIMATE)** | -| `/remove_parent_epic` | | | ✓ | Remove parent epic from epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/issues/10556)) **(ULTIMATE)** | +- Epics **(PREMIUM)** + +| Command | Issue | Merge request | Epic | Action | +| :------------------------------------ | :---- | :------------ | :--- | :------------------------------------------------------------------------------------------------------------------------------ | +| `/approve` | | ✓ | | Approve the merge request. **(STARTER)** | +| `/assign @user` | ✓ | ✓ | | Assign one user. | +| `/assign @user1 @user2` | ✓ | ✓ | | Assign multiple users. **(STARTER)** | +| `/assign me` | ✓ | ✓ | | Assign yourself. | +| `/award :emoji:` | ✓ | ✓ | ✓ | Toggle emoji award. | +| `/child_epic <epic>` | | | ✓ | Add child epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/-/issues/7330)). **(ULTIMATE)** | +| `/clear_weight` | ✓ | | | Clear weight. **(STARTER)** | +| `/close` | ✓ | ✓ | ✓ | Close. | +| `/confidential` | ✓ | | | Make confidential. | +| `/copy_metadata <!merge_request>` | ✓ | ✓ | | Copy labels and milestone from another merge request in the project. | +| `/copy_metadata <#issue>` | ✓ | ✓ | | Copy labels and milestone from another issue in the project. | +| `/create_merge_request <branch name>` | ✓ | | | Create a new merge request starting from the current issue. | +| `/done` | ✓ | ✓ | ✓ | Mark To-Do as done. | +| `/due <date>` | ✓ | | | Set due date. Examples of valid `<date>` include `in 2 days`, `this Friday` and `December 31st`. | +| `/duplicate <#issue>` | ✓ | | | Mark this issue as a duplicate of another issue and mark them as related. **(STARTER)** | +| `/epic <epic>` | ✓ | | | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. **(PREMIUM)** | +| `/estimate <<W>w <DD>d <hh>h <mm>m>` | ✓ | ✓ | | Set time estimate. For example, `/estimate 1w 3d 2h 14m`. | +| `/iteration *iteration:iteration` | ✓ | | | Set iteration ([Introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)) **(STARTER)** | +| `/label ~label1 ~label2` | ✓ | ✓ | ✓ | Add one or more labels. Label names can also start without a tilde (`~`), but mixed syntax is not supported. | +| `/lock` | ✓ | ✓ | | Lock the thread. | +| `/merge` | | ✓ | | Merge changes. Depending on the project setting, this may be [when the pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md), adding to a [Merge Train](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md), etc. | +| `/milestone %milestone` | ✓ | ✓ | | Set milestone. | +| `/move <path/to/project>` | ✓ | | | Move this issue to another project. | +| `/parent_epic <epic>` | | | ✓ | Set parent epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/issues/10556)). **(ULTIMATE)** | +| `/promote` | ✓ | | | Promote issue to epic. **(PREMIUM)** | +| `/publish` | ✓ | | | Publish issue to an associated [Status Page](status_page/index.md) ([Introduced in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906)) **(ULTIMATE)** | +| `/reassign @user1 @user2` | ✓ | ✓ | | Change assignee. **(STARTER)** | +| `/relabel ~label1 ~label2` | ✓ | ✓ | ✓ | Replace existing labels with those specified. | +| `/relate #issue1 #issue2` | ✓ | | | Mark issues as related. **(STARTER)** | +| `/remove_child_epic <epic>` | | | ✓ | Remove child epic from `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/-/issues/7330)). **(ULTIMATE)** | +| `/remove_due_date` | ✓ | | | Remove due date. | +| `/remove_epic` | ✓ | | | Remove from epic. **(PREMIUM)** | +| `/remove_estimate` | ✓ | ✓ | | Remove time estimate. | +| `/remove_iteration` | ✓ | | | Remove iteration ([Introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)) **(STARTER)** | +| `/remove_milestone` | ✓ | ✓ | | Remove milestone. | +| `/remove_parent_epic` | | | ✓ | Remove parent epic from epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/issues/10556)). **(ULTIMATE)** | +| `/remove_time_spent` | ✓ | ✓ | | Remove time spent. | +| `/remove_zoom` | ✓ | | | Remove Zoom meeting from this issue ([introduced in GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609)). | +| `/reopen` | ✓ | ✓ | ✓ | Reopen. | +| `/shrug <comment>` | ✓ | ✓ | ✓ | Append the comment with `¯\＿(ツ)＿/¯`. | +| `/spend <time(-<h>h <mm>m)> <date(<YYYY-MM-DD>)>` | ✓ | ✓ | | Subtract spent time. Optionally, specify the date that time was spent on. For example, `/spend time(-1h 30m)` or `/spend time(-1h 30m) date(2018-08-26)`. | +| `/spend <time(<h>h <mm>m)> <date(<YYYY-MM-DD>)>` | ✓ | ✓ | | Add spent time. Optionally, specify the date that time was spent on. For example, `/spend time(1h 30m)` or `/spend time(1h 30m) date(2018-08-26)`. | +| `/submit_review` | | ✓ | | Submit a pending review ([introduced in GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/issues/8041)). **(PREMIUM)** | +| `/subscribe` | ✓ | ✓ | ✓ | Subscribe to notifications. | +| `/tableflip <comment>` | ✓ | ✓ | ✓ | Append the comment with `(╯°□°)╯︵ ┻━┻`. | +| `/target_branch <local branch name>` | | ✓ | | Set target branch. | +| `/title <new title>` | ✓ | ✓ | ✓ | Change title. | +| `/todo` | ✓ | ✓ | ✓ | Add a To-Do. | +| `/unassign @user1 @user2` | ✓ | ✓ | | Remove specific assignees. **(STARTER)** | +| `/unassign` | ✓ | ✓ | | Remove all assignees. | +| `/unlabel ~label1 ~label2` or `/remove_label ~label1 ~label2` | ✓ | ✓ | ✓ | Remove all or specific labels. | +| `/unlock` | ✓ | ✓ | | Unlock the thread. | +| `/unsubscribe` | ✓ | ✓ | ✓ | Unsubscribe from notifications. | +| `/weight <value>` | ✓ | | | Set weight. Valid options for `<value>` include `0`, `1`, `2`, and so on. **(STARTER)** | +| `/wip` | | ✓ | | Toggle the Work In Progress status. | +| `/zoom <Zoom URL>` | ✓ | | | Add Zoom meeting to this issue ([introduced in GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609)). | ## Autocomplete characters @@ -86,11 +93,11 @@ to enter a parameter, compared to selecting items from a list. The easiest way to set parameters for quick actions is to use autocomplete. If you manually enter a parameter, it must be enclosed in double quotation marks -(`"`), unless it contains only: +(`"`), unless it contains only these characters: 1. ASCII letters. -1. Numerals. -1. Underscore, hyphen, question mark, dot, and ampersand. +1. Numerals (0-9). +1. Underscore (`_`), hyphen (`-`), question mark (`?`), dot (`.`), or ampersand (`&`). Parameters are also case-sensitive. Autocomplete handles this, and the insertion of quotation marks, automatically. @@ -100,7 +107,7 @@ of quotation marks, automatically. The following quick actions are applicable for commit messages: | Command | Action | -|:------------------------|:------------------------------------------| +| :---------------------- | :---------------------------------------- | | `/tag v1.2.3 <message>` | Tags this commit with an optional message | <!-- ## Troubleshooting diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index bdb99d16625..58d143fb32b 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Releases -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41766) in GitLab 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41766) in GitLab 11.7. It is typical to create a [Git tag](../../../university/training/topics/tags.md) at the moment of release to introduce a checkpoint in your source code @@ -67,9 +67,11 @@ A link is any URL which can point to whatever you like; documentation, built binaries, or other related materials. These can be both internal or external links from your GitLab instance. +The four types of links are "Runbook," "Package," "Image," and "Other." + #### Permanent links to Release assets -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27300) in GitLab 12.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9. The assets associated with a Release are accessible through a permanent URL. GitLab will always redirect this URL to the actual asset @@ -105,7 +107,7 @@ The physical location of the asset can change at any time and the direct link wi ### Releases associated with milestones -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29020) in GitLab 12.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29020) in GitLab 12.5. > - [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/39467) to edit milestones in the UI in GitLab 13.0. Releases can optionally be associated with one or more @@ -141,7 +143,7 @@ project. ### Number of Releases -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36667) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36667) in GitLab 12.8. The incremental number of Releases is displayed on the project's details page. When clicked, it takes you to the list of Releases. @@ -154,7 +156,7 @@ it is displayed to every user regardless of their permission level. ### Upcoming Releases -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/38105) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/38105) in GitLab 12.1. A Release may be created ahead of time by specifying a future `released_at` date. Until the `released_at` date and time is reached, an **Upcoming Release** badge will appear next to the @@ -186,7 +188,7 @@ we recommend doing this as one of the last steps in your CI/CD release pipeline. ## Editing a release -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/26016) in GitLab 12.6. Asset link editing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6. Asset link editing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10. To edit the details of a release, navigate to **Project overview > Releases** and click the edit button (pencil icon) in the top-right corner of the release you want to modify. @@ -205,7 +207,7 @@ through the **Edit Release** page is planned for a future version of GitLab. ## Notification for Releases -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/26001) in GitLab 12.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26001) in GitLab 12.4. You can be notified by email when a new Release is created for your project. @@ -243,7 +245,7 @@ You can also edit an existing tag to add release notes: ## Release Evidence -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/26019) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.6. Each time a release is created, GitLab takes a snapshot of data that's related to it. This data is called Release Evidence. It includes linked milestones and issues, and @@ -256,7 +258,7 @@ can have multiple Release Evidence snapshots. You can view the Release Evidence its details on the Release page. NOTE: **Note:** -When the issue tracker is disabled, release evidence [is not collected](https://gitlab.com/gitlab-org/gitlab/-/issues/208397). +When the issue tracker is disabled, release evidence [cannot be downloaded](https://gitlab.com/gitlab-org/gitlab/-/issues/208397). Release Evidence is stored as a JSON object, so you can compare evidence by using commonly-available tools. @@ -267,11 +269,16 @@ Here is an example of a Release Evidence object: { "release": { "id": 5, - "tag": "v4.0", + "tag_name": "v4.0", "name": "New release", - "project_id": 45, - "project_name": "Project name", - "released_at": "2019-06-28 13:23:40 UTC", + "project": { + "id": 20, + "name": "Project name", + "created_at": "2019-04-14T11:12:13.940Z", + "description": "Project description" + }, + "created_at": "2019-06-28 13:23:40 UTC", + "description": "Release description", "milestones": [ { "id": 11, @@ -311,7 +318,7 @@ Here is an example of a Release Evidence object: } ``` -### Enabling Release Evidence display **(CORE ONLY)** +### Diabling Release Evidence display **(CORE ONLY)** This feature comes with the `:release_evidence_collection` feature flag enabled by default in GitLab self-managed instances. To turn it off, @@ -393,6 +400,8 @@ deploy_to_production: - if: $CI_DEPLOY_FREEZE == null ``` +For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md). + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/repository/forking_workflow.md b/doc/user/project/repository/forking_workflow.md index a49701017f3..75a84e36169 100644 --- a/doc/user/project/repository/forking_workflow.md +++ b/doc/user/project/repository/forking_workflow.md @@ -46,7 +46,7 @@ You can use [repository mirroring](repository_mirroring.md) to keep your fork sy The main difference is that with repository mirroring your remote fork will be automatically kept up-to-date. -Without mirroring, to work locally you'll have to use `git pull` to update your local repo +Without mirroring, to work locally you'll have to use `git pull` to update your local repository with the upstream project, then push the changes back to your fork to update it. CAUTION: **Caution:** diff --git a/doc/user/project/repository/git_blame.md b/doc/user/project/repository/git_blame.md index 2deb53b313c..e63b57747ef 100644 --- a/doc/user/project/repository/git_blame.md +++ b/doc/user/project/repository/git_blame.md @@ -25,7 +25,7 @@ for that commit. ## Blame previous commit -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/19299) in GitLab 12.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/19299) in GitLab 12.7. To see earlier revisions of a specific line, click **View blame prior to this change** until you've found the changes you're interested in viewing: diff --git a/doc/user/project/repository/img/repository_cleanup.png b/doc/user/project/repository/img/repository_cleanup.png Binary files differdeleted file mode 100644 index e343f23ac27..00000000000 --- a/doc/user/project/repository/img/repository_cleanup.png +++ /dev/null diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index 055443daa1f..48975b7864e 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -27,7 +27,7 @@ that you [connect with GitLab via SSH](../../../ssh/README.md). ## Files -Use a repository to store your files in GitLab. From [GitLab 12.10 onwards](https://gitlab.com/gitlab-org/gitlab/issues/33806), +Use a repository to store your files in GitLab. In [GitLab 12.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33806), you'll see on the repository's file tree an icon next to the file name according to its extension: @@ -84,9 +84,9 @@ according to the markup language. | [AsciiDoc](../../asciidoc.md) | `adoc`, `ad`, `asciidoc` | | [Textile](https://textile-lang.com/) | `textile` | | [rdoc](http://rdoc.sourceforge.net/doc/index.html) | `rdoc` | -| [Orgmode](https://orgmode.org/) | `org` | +| [Org mode](https://orgmode.org/) | `org` | | [creole](http://www.wikicreole.org/) | `creole` | -| [Mediawiki](https://www.mediawiki.org/wiki/MediaWiki) | `wiki`, `mediawiki` | +| [MediaWiki](https://www.mediawiki.org/wiki/MediaWiki) | `wiki`, `mediawiki` | ### Repository README and index files @@ -116,7 +116,7 @@ user's sessions and include code, narrative text, equations, and rich output. ### OpenAPI viewer -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/19515) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/19515) in GitLab 12.6. GitLab can render OpenAPI specification files with its file viewer, provided their filenames include `openapi` or `swagger` and their extension is `yaml`, @@ -219,7 +219,9 @@ vendored code, and most markup languages are excluded. This behavior can be adjusted by overriding the default. For example, to enable `.proto` files to be detected, add the following to `.gitattributes` in the root of your repository. -> *.proto linguist-detectable=true +```plaintext +*.proto linguist-detectable=true +``` ## Locked files **(PREMIUM)** @@ -232,7 +234,7 @@ You can access your repos via [repository API](../../../api/repositories.md). ## Clone in Apple Xcode -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/45820) in GitLab 11.0 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45820) in GitLab 11.0 Projects that contain a `.xcodeproj` or `.xcworkspace` directory can now be cloned in Xcode using the new **Open in Xcode** button, located next to the Git URL @@ -240,7 +242,7 @@ used for cloning your project. The button is only shown on macOS. ## Download Source Code -> Support for directory download was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/24704) in GitLab 11.11. +> Support for directory download was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/24704) in GitLab 11.11. The source code stored in a repository can be downloaded from the UI. By clicking the download icon, a dropdown will open with links to download the following: diff --git a/doc/user/project/repository/jupyter_notebooks/index.md b/doc/user/project/repository/jupyter_notebooks/index.md index ca82be280d9..1948b12aacd 100644 --- a/doc/user/project/repository/jupyter_notebooks/index.md +++ b/doc/user/project/repository/jupyter_notebooks/index.md @@ -1,6 +1,6 @@ # Jupyter Notebook Files -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/2508/) in GitLab 9.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/2508/) in GitLab 9.1. [Jupyter](https://jupyter.org/) Notebook (previously IPython Notebook) files are used for interactive computing in many fields and contain a complete record of the diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md index 16bffe5417d..124150c441a 100644 --- a/doc/user/project/repository/reducing_the_repo_size_using_git.md +++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md @@ -1,150 +1,244 @@ --- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers type: howto --- -# Reducing the repository size using Git - -A GitLab Enterprise Edition administrator can set a [repository size limit](../../admin_area/settings/account_and_limit_settings.md) -which will prevent you from exceeding it. - -When a project has reached its size limit, you will not be able to push to it, -create a new merge request, or merge existing ones. You will still be able to -create new issues, and clone the project though. Uploading LFS objects will -also be denied. - -If you exceed the repository size limit, your first thought might be to remove -some data, make a new commit and push back to the repository. Perhaps you can -move some blobs to LFS, or remove some old dependency updates from history. -Unfortunately, it's not so easy and that workflow won't work. Deleting files in -a commit doesn't actually reduce the size of the repo since the earlier commits -and blobs are still around. What you need to do is rewrite history with Git's -[`filter-branch` option](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#The-Nuclear-Option:-filter-branch), -or an open source community-maintained tool like the -[BFG](https://rtyley.github.io/bfg-repo-cleaner/). - -Note that even with that method, until `git gc` runs on the GitLab side, the -"removed" commits and blobs will still be around. You also need to be able to -push the rewritten history to GitLab, which may be impossible if you've already -exceeded the maximum size limit. +# Reduce repository size -In order to lift these restrictions, the administrator of the GitLab instance -needs to increase the limit on the particular project that exceeded it, so it's -always better to spot that you're approaching the limit and act proactively to -stay underneath it. If you hit the limit, and your admin can't - or won't - -temporarily increase it for you, your only option is to prune all the unneeded -stuff locally, and then create a new project on GitLab and start using that -instead. +Git repositories become larger over time. When large files are added to a Git repository: -If you can continue to use the original project, we recommend [using -BFG](#using-the-bfg-repo-cleaner), a tool that's built and -maintained by the open source community. It's faster and simpler than -`git filter-branch`, and GitLab can use its account of what has changed to clean -up its own internal state, maximizing the space saved. +- Fetching the repository becomes slower because everyone must download the files. +- They take up a large amount of storage space on the server. +- Git repository storage limits [can be reached](#storage-limits). -CAUTION: **Caution:** -Make sure to first make a copy of your repository since rewriting history will -purge the files and information you are about to delete. Also make sure to -inform any collaborators to not use `pull` after your changes, but use `rebase`. +Rewriting a repository can remove unwanted history to make the repository smaller. +[`git filter-repo`](https://github.com/newren/git-filter-repo) is a tool for quickly rewriting Git +repository history, and is recommended over both: -CAUTION: **Caution:** -This process is not suitable for removing sensitive data like password or keys -from your repository. Information about commits, including file content, is -cached in the database, and will remain visible even after they have been -removed from the repository. +- [`git filter-branch`](https://git-scm.com/docs/git-filter-branch). +- [BFG](https://rtyley.github.io/bfg-repo-cleaner/). + +DANGER: **Danger:** +Rewriting repository history is a destructive operation. Make sure to backup your repository before +you begin. The best way back up a repository is to +[export the project](../settings/import_export.md#exporting-a-project-and-its-data). -## Using the BFG Repo-Cleaner +## Purge files from repository history -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/19376) in GitLab 11.6. +To make cloning your project faster, rewrite branches and tags to remove unwanted files. -1. [Install BFG](https://rtyley.github.io/bfg-repo-cleaner/) from its open source community repository. +1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/master/INSTALL.md) + using a supported package manager or from source. -1. Navigate to your repository: +1. Clone a fresh copy of the repository using `--bare`: ```shell - cd my_repository/ + git clone --bare https://example.gitlab.com/my/project.git ``` -1. Change to the branch you want to remove the big file from: +1. Using `git filter-repo`, purge any files from the history of your repository. + + To purge all large files, the `--strip-blobs-bigger-than` option can be used: ```shell - git checkout master + git filter-repo --strip-blobs-bigger-than 10M ``` -1. Create a commit removing the large file from the branch, if it still exists: + To purge specific large files by path, the `--path` and `--invert-paths` options can be combined: ```shell - git rm path/to/big_file.mpg - git commit -m 'Remove unneeded large file' + git filter-repo --path path/to/big/file.m4v --invert-paths ``` -1. Rewrite history: + See the + [`git filter-repo` documentation](https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#EXAMPLES) + for more examples and the complete documentation. + +1. Running `git filter-repo` removes all remotes. To restore the remote for your project, run: ```shell - bfg --delete-files path/to/big_file.mpg + git remote add origin https://example.gitlab.com/<namespace>/<project_name>.git ``` - An object map file will be written to `object-id-map.old-new.txt`. Keep it - around - you'll need it for the final step! +1. Force push your changes to overwrite all branches on GitLab: -1. Force-push the changes to GitLab: + ```shell + git push origin --force --all + ``` + + [Protected branches](../protected_branches.md) will cause this to fail. To proceed, you must + remove branch protection, push, and then re-enable protected branches. + +1. To remove large files from tagged releases, force push your changes to all tags on GitLab: ```shell - git push --force-with-lease origin master + git push origin --force --tags ``` - If this step fails, someone has changed the `master` branch while you were - rewriting history. You could restore the branch and re-run BFG to preserve - their changes, or use `git push --force` to overwrite their changes. + [Protected tags](../protected_tags.md) will cause this to fail. To proceed, you must remove tag + protection, push, and then re-enable protected tags. -1. Navigate to **Project > Settings > Repository > Repository Cleanup**: +## Purge files from GitLab storage -  +To reduce the size of your repository in GitLab, you must remove GitLab internal references to +commits that contain large files. Before completing these steps, +[purge files from your repository history](#purge-files-from-repository-history). - Upload the `object-id-map.old-new.txt` file and press **Start cleanup**. - This will remove any internal Git references to the old commits, and run - `git gc` against the repository. You will receive an email once it has - completed. +As well as [branches](branches/index.md) and tags, which are a type of Git ref, GitLab automatically +creates other refs. These refs prevent dead links to commits, or missing diffs when viewing merge +requests. [Repository cleanup](#repository-cleanup) can be used to remove these from GitLab. -NOTE: **Note:** -This process will remove some copies of the rewritten commits from GitLab's -cache and database, but there are still numerous gaps in coverage - at present, -some of the copies may persist indefinitely. [Clearing the instance cache](../../../administration/raketasks/maintenance.md#clear-redis-cache) -may help to remove some of them, but it should not be depended on for security -purposes! +The following internal refs are not advertised: -## Using `git filter-branch` +- `refs/merge-requests/*` for merge requests. +- `refs/pipelines/*` for + [pipelines](../../../ci/pipelines/index.md#troubleshooting-fatal-reference-is-not-a-tree). +- `refs/environments/*` for environments. -1. Navigate to your repository: +This means they are not usually included when fetching, which makes fetching faster. In addition, +`refs/keep-around/*` are hidden refs to prevent commits with discussion from being deleted and +cannot be fetched at all. - ```shell - cd my_repository/ - ``` +However, these refs can be accessed from the Git bundle inside a project export. -1. Change to the branch you want to remove the big file from: +1. [Install `git filter-repo`](https://github.com/newren/git-filter-repo/blob/master/INSTALL.md) + using a supported package manager or from source. + +1. Generate a fresh [export from the + project](../settings/import_export.html#exporting-a-project-and-its-data) and download it. + +1. Decompress the backup using `tar`: ```shell - git checkout master + tar xzf project-backup.tar.gz ``` -1. Use `filter-branch` to remove the big file: + This will contain a `project.bundle` file, which was created by + [`git bundle`](https://git-scm.com/docs/git-bundle). + +1. Clone a fresh copy of the repository from the bundle: ```shell - git filter-branch --force --tree-filter 'rm -f path/to/big_file.mpg' HEAD + git clone --bare --mirror /path/to/project.bundle ``` -1. Instruct Git to purge the unwanted data: +1. Using `git filter-repo`, purge any files from the history of your repository. Because we are + trying to remove internal refs, we will rely on the `commit-map` produced by each run to tell us + which internal refs to remove. + + NOTE:**Note:** + `git filter-repo` creates a new `commit-map` file every run, and overwrite the `commit-map` from + the previous run. You will need this file from **every** run. Do the next step every time you run + `git filter-repo`. + + To purge all large files, the `--strip-blobs-bigger-than` option can be used: ```shell - git reflog expire --expire=now --all && git gc --prune=now --aggressive + git filter-repo --strip-blobs-bigger-than 10M ``` -1. Lastly, force push to the repository: + To purge specific large files by path, the `--path` and `--invert-paths` options can be combined. ```shell - git push --force origin master + git filter-repo --path path/to/big/file.m4v --invert-paths ``` -Your repository should now be below the size limit. + See the + [`git filter-repo` documentation](https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#EXAMPLES) + for more examples and the complete documentation. + +1. Run a [repository cleanup](#repository-cleanup). + +## Repository cleanup + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/19376) in GitLab 11.6. + +Repository cleanup allows you to upload a text file of objects and GitLab will remove internal Git +references to these objects. You can use +[`git filter-repo`](https://github.com/newren/git-filter-repo) to produce a list of objects (in a +`commit-map` file) that can be used with repository cleanup. + +To clean up a repository: + +1. Go to the project for the repository. +1. Navigate to **{settings}** **Settings > Repository**. +1. Upload a list of objects. For example, a `commit-map` file. +1. Click **Start cleanup**. + +This will: + +- Remove any internal Git references to old commits. +- Run `git gc` against the repository. + +You will receive an email once it has completed. + +When using repository cleanup, note: + +- Housekeeping prunes loose objects older than 2 weeks. This means objects added in the last 2 weeks + will not be removed immediately. If you have access to the + [Gitaly](../../../administration/gitaly/index.md) server, you may run `git gc --prune=now` to + prune all loose objects immediately. +- This process will remove some copies of the rewritten commits from GitLab's cache and database, + but there are still numerous gaps in coverage and some of the copies may persist indefinitely. + [Clearing the instance cache](../../../administration/raketasks/maintenance.md#clear-redis-cache) + may help to remove some of them, but it should not be depended on for security purposes! + +## Storage limits + +Repository size limits: + +- Can [be set by an administrator](../../admin_area/settings/account_and_limit_settings.md#repository-size-limit-starter-only) + on self-managed instances. **(STARTER ONLY)** +- Are [set for GitLab.com](../../gitlab_com/index.md#repository-size-limit). + +When a project has reached its size limit, you cannot: + +- Push to the project. +- Create a new merge request. +- Merge existing merge requests. +- Upload LFS objects. + +You can still: + +- Create new issues. +- Clone the project. + +If you exceed the repository size limit, you might try to: + +1. Remove some data. +1. Make a new commit. +1. Push back to the repository. + +Perhaps you might also: + +- Move some blobs to LFS. +- Remove some old dependency updates from history. + +Unfortunately, this workflow won't work. Deleting files in a commit doesn't actually reduce the size +of the repository because the earlier commits and blobs still exist. + +What you need to do is rewrite history. We recommend the open-source community-maintained tool +[`git filter-repo`](https://github.com/newren/git-filter-repo). + +NOTE: **Note:** +Until `git gc` runs on the GitLab side, the "removed" commits and blobs will still exist. You also +must be able to push the rewritten history to GitLab, which may be impossible if you've already +exceeded the maximum size limit. + +In order to lift these restrictions, the administrator of the self-managed GitLab instance must +increase the limit on the particular project that exceeded it. Therefore, it's always better to +proactively stay underneath the limit. If you hit the limit, and can't have it temporarily +increased, your only option is to: + +1. Prune all the unneeded stuff locally. +1. Create a new project on GitLab and start using that instead. + +CAUTION: **Caution:** +This process is not suitable for removing sensitive data like password or keys from your repository. +Information about commits, including file content, is cached in the database, and will remain +visible even after they have been removed from the repository. <!-- ## Troubleshooting diff --git a/doc/user/project/repository/repository_mirroring.md b/doc/user/project/repository/repository_mirroring.md index fdbea385998..f75b083e6dc 100644 --- a/doc/user/project/repository/repository_mirroring.md +++ b/doc/user/project/repository/repository_mirroring.md @@ -28,7 +28,7 @@ immediate update, unless: - The mirror is already being updated. - 5 minutes haven't elapsed since its last update. -For security reasons, from [GitLab 12.10 onwards](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27166), +For security reasons, in [GitLab 12.10 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27166), the URL to the original repository is only displayed to users with Maintainer or Owner permissions to the mirrored project. @@ -134,7 +134,7 @@ The repository will push soon. To force a push, click the appropriate button. ## Pulling from a remote repository **(STARTER)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51) in GitLab Enterprise Edition 8.2. -> - [Added Git LFS support](https://gitlab.com/gitlab-org/gitlab/issues/10871) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.11. +> - [Added Git LFS support](https://gitlab.com/gitlab-org/gitlab/-/issues/10871) in [GitLab Starter](https://about.gitlab.com/pricing/) 11.11. You can set up a repository to automatically have its branches, tags, and commits updated from an upstream repository. @@ -356,6 +356,24 @@ a [Push event webhook](../integrations/webhooks.md#push-events) to trigger an im pull to GitLab. Push mirroring from GitLab is rate limited to once per minute when only push mirroring protected branches. +### Configure a webhook to trigger an immediate pull to GitLab + +Assuming you have already configured the [push](#setting-up-a-push-mirror-to-another-gitlab-instance-with-2fa-activated) and [pull](#pulling-from-a-remote-repository-starter) mirrors in the upstream GitLab instance, to trigger an immediate pull as suggested above, you will need to configure a [Push Event Web Hook](../integrations/webhooks.md#push-events) in the downstream instance. + +To do this: + +- Create a [personal access token](../../profile/personal_access_tokens.md) with `API` scope. +- Navigate to **Settings > Webhooks** +- Add the webhook URL which in this case will use the [Pull Mirror API](../../../api/projects.md#start-the-pull-mirroring-process-for-a-project-starter) request to trigger an immediate pull after updates to the repository. + + ```plaintext + https://gitlab.example.com/api/v4/projects/:id/mirror/pull?private_token=<your_access_token> + ``` + +- Ensure that the **Push Events** checkbox is selected. +- Click on **Add Webhook** button to save the webhook. +- To test the integration click on the **Test** button and confirm GitLab does not return any error. + ### Preventing conflicts using a `pre-receive` hook CAUTION: **Warning:** @@ -388,13 +406,13 @@ proxy_push() REFNAME="$3" # --- Pattern of branches to proxy pushes - whitelisted=$(expr "$branch" : "\(master\)") + allowlist=$(expr "$branch" : "\(master\)") case "$refname" in refs/heads/*) branch=$(expr "$refname" : "refs/heads/\(.*\)") - if [ "$whitelisted" = "$branch" ]; then + if [ "$allowlist" = "$branch" ]; then unset GIT_QUARANTINE_PATH # handle https://git-scm.com/docs/git-receive-pack#_quarantine_environment error="$(git push --quiet $TARGET_REPO $NEWREV:$REFNAME 2>&1)" fail=$? @@ -435,7 +453,7 @@ Note that this sample has a few limitations: - This example may not work verbatim for your use case and might need modification. - It does not regard different types of authentication mechanisms for the mirror. - It does not work with forced updates (rewriting history). - - Only branches that match the `whitelisted` patterns will be proxy pushed. + - Only branches that match the `allowlist` patterns will be proxy pushed. - The script circumvents the Git hook quarantine environment because the update of `$TARGET_REPO` is seen as a ref update and Git will complain about it. diff --git a/doc/user/project/repository/x509_signed_commits/index.md b/doc/user/project/repository/x509_signed_commits/index.md index 20143af0b33..d55d5c5c2d8 100644 --- a/doc/user/project/repository/x509_signed_commits/index.md +++ b/doc/user/project/repository/x509_signed_commits/index.md @@ -65,11 +65,11 @@ git config --global gpg.format x509 ### Windows and MacOS -Install [smimesign](https://github.com/github/smimesign) by downloading the +Install [S/MIME Sign](https://github.com/github/smimesign) by downloading the installer or via `brew install smimesign` on MacOS. Get the ID of your certificate with `smimesign --list-keys` and set your -signingkey `git config --global user.signingkey ID`, then configure X.509: +signing key `git config --global user.signingkey ID`, then configure X.509: ```shell git config --global gpg.x509.program smimesign diff --git a/doc/user/project/requirements/img/requirement_edit_save_v12_10.png b/doc/user/project/requirements/img/requirement_edit_save_v12_10.png Binary files differdeleted file mode 100644 index 6cf7db361b8..00000000000 --- a/doc/user/project/requirements/img/requirement_edit_save_v12_10.png +++ /dev/null diff --git a/doc/user/project/requirements/index.md b/doc/user/project/requirements/index.md index 50343e52a68..d9bd02518a4 100644 --- a/doc/user/project/requirements/index.md +++ b/doc/user/project/requirements/index.md @@ -1,14 +1,23 @@ --- type: reference, howto +stage: Plan +group: Certify +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Requirements **(ULTIMATE)** +# Requirements Management **(ULTIMATE)** > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2703) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.10. -Requirements allow you to create criteria to check your products against. They -can be based on users, stakeholders, system, software, or anything else you -find important to capture. +With requirements, you can set criteria to check your products against. They can be based on users, +stakeholders, system, software, or anything else you find important to capture. + +A requirement is an artifact in GitLab which describes the specific behavior of your product. +Requirements are long-lived and don't disappear unless manually cleared. + +If an industry standard *requires* that your application has a certain feature or behavior, you can +[create a requirement](#create-a-requirement) to reflect this. +When a feature is no longer necessary, you can [archive the related requirement](#archive-a-requirement). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [GitLab 12.10 Introduces Requirements Management](https://www.youtube.com/watch?v=uSS7oUNSEoU). @@ -38,22 +47,18 @@ list page. To edit a requirement: -1. From the requirements list, click the **Edit** (**{pencil}**) button. +1. From the requirements list, click **Edit** (**{pencil}**). 1. Update the title in text input field. 1. Click **Save changes**.  -The requirements list shows the new title immediately. - - - ## Archive a requirement You can archive an open requirement (if you have the necessary privileges) while you're in the **Open** tab. -From the requirements list page, click the **Archive** (**{archive}**) button. +From the requirements list page, click **Archive** (**{archive}**).  @@ -65,6 +70,84 @@ You can view the list of archived requirements in the **Archived** tab.  -To reopen an archived requirement, click the **Reopen** button. +To reopen an archived requirement, click **Reopen**. As soon as a requirement is reopened, it no longer appears in the **Archived** tab. + +## Search for a requirement from the requirements list page + +> - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. + +You can search for a requirement from the list of requirements using filtered search bar (similar to +that of Issues and Merge Requests) based on following parameters: + +- Title +- Author username + +To search, go to the list of requirements and click the field **Search or filter results**. +It will display a dropdown menu, from which you can add an author. You can also enter plain +text to search by epic title or description. When done, press <kbd>Enter</kbd> on your +keyboard to filter the list. + +You can also sort requirements list by: + +- Created date +- Last updated + +## Allow requirements to be satisfied from a CI job + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2859) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. + +GitLab supports [requirements test +reports](../../../ci/pipelines/job_artifacts.md#artifactsreportsrequirements-ultimate) now. +You can add a job to your CI pipeline that, when triggered, marks all existing +requirements as Satisfied. + +### Add the manual job to CI + +To configure your CI to mark requirements as Satisfied when the manual job is +triggered, add the code below to your `.gitlab-ci.yml` file. + +```yaml +requirements_confirmation: + when: manual + allow_failure: false + script: + - mkdir tmp + - echo "{\"*\":\"passed\"}" > tmp/requirements.json + artifacts: + reports: + requirements: tmp/requirements.json +``` + +This definition adds a manually-triggered (`when: manual`) job to the CI +pipeline. It's blocking (`allow_failure: false`), but it's up to you what +conditions you use for triggering the CI job. Also, you can use any existing CI job +to mark all requirements as satisfied, as long as the `requirements.json` +artifact is generated and uploaded by the CI job. + +When you manually trigger this job, the `requirements.json` file containing +`{"*":"passed"}` is uploaded as an artifact to the server. On the server side, +the requirement report is checked for the "all passed" record +(`{"*":"passed"}`), and on success, it marks all existing open requirements as +Satisfied. + +### Add the manual job to CI conditionally + +To configure your CI to include the manual job only when there are some open +requirements, add a rule which checks `CI_HAS_OPEN_REQUIREMENTS` CI variable. + +```yaml +requirements_confirmation: + rules: + - if: "$CI_HAS_OPEN_REQUIREMENTS" == "true" + when: manual + - when: never + allow_failure: false + script: + - mkdir tmp + - echo "{\"*\":\"passed\"}" > tmp/requirements.json + artifacts: + reports: + requirements: tmp/requirements.json +``` diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index d021f259015..ffb1f6a1407 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -1,6 +1,13 @@ +--- +stage: Plan +group: Certify +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + # Service Desk **(STARTER)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/149) in [GitLab Premium 9.1](https://about.gitlab.com/releases/2017/04/22/gitlab-9-1-released/#service-desk-eep). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/149) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.1. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/214839) to [GitLab Starter](https://about.gitlab.com/pricing/) in 13.0. ## Overview @@ -28,14 +35,19 @@ with GitLab CI/CD. Here's how Service Desk will work for you: -1. You'll provide a project-specific email address to your paying customers, who can email you directly from within the app -1. Each email they send creates an issue in the appropriate project -1. Your team members navigate to the Service Desk issue tracker, where they can see new support requests and respond inside associated issues -1. Your team communicates back and forth with the customer to understand the request -1. Your team starts working on implementing code to solve your customer's problem -1. When your team finishes the implementation, whereupon the merge request is merged and the issue is closed automatically -1. The customer will have been attended successfully via email, without having real access to your GitLab instance -1. Your team saved time by not having to leave GitLab (or setup any integrations) to follow up with your customer +1. You provide a project-specific email address to your paying customers, who can email you directly + from within the app. +1. Each email they send creates an issue in the appropriate project. +1. Your team members navigate to the Service Desk issue tracker, where they can see new support + requests and respond inside associated issues. +1. Your team communicates back and forth with the customer to understand the request. +1. Your team starts working on implementing code to solve your customer's problem. +1. When your team finishes the implementation, whereupon the merge request is merged and the issue + is closed automatically. +1. The customer will have been attended successfully via email, without having real access to your + GitLab instance. +1. Your team saved time by not having to leave GitLab (or setup any integrations) to follow up with + your customer. ## How it works @@ -56,7 +68,8 @@ If you have the correct access and a Premium license, you have the option to set Follow these steps to do so: 1. [Set up incoming email](../../administration/incoming_email.md#set-it-up) for the GitLab instance. - This must support [email sub-addressing](../../administration/incoming_email.md#email-sub-addressing). + - We recommend using [email sub-addressing](../../administration/incoming_email.md#email-sub-addressing), + but in GitLab 11.7 and later you can also use [catch-all mailboxes](../../administration/incoming_email.md#catch-all-mailbox). 1. Navigate to your project's **Settings > General** and locate the **Service Desk** section. 1. Enable the **Activate Service Desk** toggle. This reveals a unique email address to email issues to the project. These issues will be [confidential](issues/confidential_issues.md), so they will @@ -83,7 +96,7 @@ navigation's **Issues** menu. ### Using customized email templates - > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2460) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. + > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2460) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.7. When a user submits a new issue using Service Desk, or when a new note is created on a Service Desk issue, an email is sent to the author. @@ -110,14 +123,14 @@ in the email, `%{ISSUE_PATH}` placeholder which will be replaced by ### Using custom email display name -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7529) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7529) in GitLab 12.8. You can customize the email display name. Emails sent from Service Desk will have this name in the `From` header. The default display name is `GitLab Support Bot`. ### Using custom email address -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2201) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2201) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0. NOTE: **Note:** This feature is disabled by default. For steps to enable it, see [Enable custom email address](#enable-custom-email-address). @@ -160,12 +173,12 @@ As a result, a new Service Desk issue is created from this email in the `mygroup #### Enable custom email address -This feature comes with the `service_desk_email` feature flag disabled by default. +This feature comes with the `service_desk_custom_address` feature flag disabled by default. To turn on the feature, ask a GitLab administrator with Rails console access to run the following command: ```ruby -Feature.enable(service_desk_email) +Feature.enable(:service_desk_custom_address) ``` The configuration options are the same as for configuring diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index e9521a0567e..5a364eb89aa 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -1,6 +1,6 @@ # Project import/export -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/3050) in GitLab 8.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3050) in GitLab 8.9. > - From GitLab 10.0, administrators can disable the project export option on the GitLab instance. Existing projects running on any GitLab instance or GitLab.com can be exported with all their related @@ -158,12 +158,16 @@ If use of the `Internal` visibility level [is restricted](../../../public_access/public_access.md#restricting-the-use-of-public-or-internal-projects), all imported projects are given the visibility of `Private`. +NOTE: **Note:** +The maximum import file size can be set by the Administrator, default is 50MB. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin UI](../../admin_area/settings/account_and_limit_settings.md). + ## Rate limits To help avoid abuse, users are rate limited to: -| Request Type | Limit | -| ---------------- | --------------------------- | -| Export | 1 project per 5 minutes | -| Download export | 10 projects per 10 minutes | -| Import | 30 projects per 5 minutes | +| Request Type | Limit | +| ---------------- | ----------------------------------------- | +| Export | 30 projects per 5 minutes | +| Download export | 10 downloads per project every 10 minutes | +| Import | 30 projects per 5 minutes | diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 0c98772237b..0798c39fff5 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -55,7 +55,7 @@ Use the switches to enable or disable the following features: | **Merge Requests** | ✓ | Enables [merge request](../merge_requests/) functionality; also see [Merge request settings](#merge-request-settings) | | **Forks** | ✓ | Enables [forking](../index.md#fork-a-project) functionality | | **Pipelines** | ✓ | Enables [CI/CD](../../../ci/README.md) functionality | -| **Container Registry** | | Activates a [registry](../../packages/container_registry/) for your docker images | +| **Container Registry** | | Activates a [registry](../../packages/container_registry/) for your Docker images | | **Git Large File Storage** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-large-file-storage-lfs) | | **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration-premium-only) functionality | | **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/) | @@ -192,11 +192,9 @@ to transfer a project. You can transfer an existing project into a [group](../../group/index.md) if: -1. You have at least **Maintainer** [permissions](../../permissions.md#project-members-permissions) to that group. -1. The project is in a subgroup you own. -1. You're at least a **Maintainer** of the project under your personal namespace. - Similarly, if you're an owner of a group, you can transfer any of its projects - under your own user. +- You have at least **Maintainer** [permissions](../../permissions.md#project-members-permissions) to that group. +- You're at least an **Owner** of the project to be transferred. +- The group to which the project is being transferred to must allow creation of new projects. To transfer a project: @@ -228,14 +226,14 @@ To remove a project: This action either: - Removes a project including all associated resources (issues, merge requests etc). -- Since [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/32935), on +- Since [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/32935), on [GitLab Premium or GitLab.com Silver](https://about.gitlab.com/pricing/) or higher tiers, marks a project for deletion. The deletion will happen 7 days later by default, but this can be changed in the [instance settings](../../admin_area/settings/visibility_and_access_controls.md#default-deletion-adjourned-period-premium-only). #### Restore a project **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/32935) in GitLab 12.6. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. To restore a project marked for deletion: @@ -272,3 +270,8 @@ Configure Error Tracking to discover and view [Sentry errors within GitLab](../o ### Jaeger tracing **(ULTIMATE)** Add the URL of a Jaeger server to allow your users to [easily access the Jaeger UI from within GitLab](../operations/tracing.md). + +### Status Page + +[Add Storage credentials](../status_page/#syncing-incidents-to-the-status-page) +to enable the syncing of public Issues to a [deployed status page](../status_page/#status-page-project). diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index 303a6f6d3be..42ba2654b42 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -1,6 +1,13 @@ -# Project access tokens **(CORE ONLY)** +# Project access tokens (Alpha) **(CORE ONLY)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2587) in GitLab 13.0. +CAUTION: **Warning:** +This is an [Alpha](https://about.gitlab.com/handbook/product/#alpha) feature, and it is subject to change at any time without +prior notice. + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2587) in GitLab 13.0. +> - It's deployed behind a feature flag, disabled by default. +> - It's disabled on GitLab.com. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-project-access-tokens). Project access tokens are scoped to a project and can be used to authenticate with the [GitLab API](../../../api/README.md#personalproject-access-tokens). @@ -27,7 +34,7 @@ For each project access token created, a bot user will also be created and added ["Maintainer" level permissions](../../permissions.md#project-members-permissions). API calls made with a project access token will be associated to the corresponding bot user. -These users will appear in **{settings}** **Settings > Members** but can not be modified. +These users will appear in **Members** but can not be modified. Furthermore, the bot user can not be added to any other project. When the project access token is [revoked](#revoking-a-project-access-token) the bot user will be deleted and all @@ -53,3 +60,22 @@ the following table. | `write_registry` | Allows write-access (push) to [container registry](../../packages/container_registry/index.md). | | `read_repository` | Allows read-only access (pull) to the repository. | | `write_repository` | Allows read-write access (pull, push) to the repository. | + +### Enable or disable project access tokens + +Project access tokens is an [Alpha](https://about.gitlab.com/handbook/product/#alpha) feature and is not recommended for production use. +It is deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) +can enable it for your instance. + +To enable it: + +```ruby +Feature.enable(:resource_access_token) +``` + +To disable it: + +```ruby +Feature.disable(:resource_access_token) +``` diff --git a/doc/user/project/status_page/index.md b/doc/user/project/status_page/index.md index 8ebfb638894..ec0a79583d5 100644 --- a/doc/user/project/status_page/index.md +++ b/doc/user/project/status_page/index.md @@ -55,7 +55,7 @@ To deploy the Status Page to AWS S3 you need to add the Status Page project & co Once the CI/CD variables are set, you'll need to set up the Project you want to use for Incident issues: -1. Navigate to **Settings > Operations > Status Page**. +1. To view the [Operations Settings](../settings/#operations-settings) page, navigate to **{settings}** **Settings > Operations > Status Page**. 1. Fill in your cloud provider's credentials and make sure the **Active** checkbox is checked. 1. Click **Save changes**. @@ -71,7 +71,8 @@ The incident detail page shows detailed information about a particular incident. - Status on the incident, including when the incident was last updated. - The incident title, including any emojis. -- The description of the incident, including emojis and static images. +- The description of the incident, including emojis. +- Any file attachments provided in the incident description or comments with a valid image extension. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.1. - A chronological ordered list of updates to the incident.  @@ -82,12 +83,21 @@ The incident detail page shows detailed information about a particular incident. To publish an Incident, you first need to create an issue in the Project you enabled the Status Page settings in. -Once this issue is created, a background worker will publish the issue onto the Status Page using the credentials you provided during setup. +Issues are not published to the Status Page by default. Use the `/publish` [quick action](../quick_actions.md) in an issue to publish the issue. Only [project or group owners](../../permissions.md) are permitted to publish issues. + +After the quick action is used, a background worker publishes the issue onto the Status Page using the credentials you provided during setup. + Since all incidents are published publicly, user and group mentions are anonymized with `Incident Responder`, and titles of non-public [GitLab references](../../markdown.md#special-gitlab-references) are removed. +When an Incident is published in the GitLab project, you can access the +details page of the Incident by clicking the **Published on status page** button +displayed under the Incident's title. + + + NOTE: **Note:** -Confidential issues are not published. If a published issue is made confidential it will be unpublished. +Confidential issues can't be published. If you make a published issue confidential, it will be unpublished. ### Publishing updates @@ -108,3 +118,15 @@ Anyone with access to view the Issue can add an Emoji Award to a comment, so you ### Changing the Incident status To change the incident status from `open` to `closed`, close the incident issue within GitLab. This will then be updated shortly on the Status Page website. + +## Attachment storage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.1. + +Beginning with GitLab 13.1, files attached to incident issue descriptions or +comments are published and unpublished to the status page storage as part of +the [publication flow](#how-it-works). + +### Limit + +Only 5000 attachments per issue will be transferred to the status page. diff --git a/doc/user/project/time_tracking.md b/doc/user/project/time_tracking.md index 57a26f4e928..b88e1ed2d37 100644 --- a/doc/user/project/time_tracking.md +++ b/doc/user/project/time_tracking.md @@ -1,6 +1,9 @@ --- type: reference disqus_identifier: 'https://docs.gitlab.com/ee/workflow/time_tracking.html' +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # Time Tracking diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index d4daca0e1e4..0ddc9762bc5 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -1,7 +1,7 @@ # Web IDE -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4539) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. -> - [Brought to GitLab Core](https://gitlab.com/gitlab-org/gitlab-foss/issues/44157) in 10.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4539) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/44157) to GitLab Core in 10.7. The Web IDE editor makes it faster and easier to contribute changes to your projects by providing an advanced editor with commit staging. @@ -57,10 +57,30 @@ which applies to the entire Web IDE screen. |---------------------------------------------------------------|-----------------------------------------| |  |  | +## Configure the Web IDE + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23352) in [GitLab Core](https://about.gitlab.com/pricing/) 13.1. + +The Web IDE supports configuration of certain editor settings by using +[`.editorconfig` files](https://editorconfig.org/). When opening a file, the +Web IDE looks for a file named `.editorconfig` in the current directory +and all parent directories. If a configuration file is found and has settings +that match the file's path, these settings will be enforced on the opened file. + +The Web IDE currently supports the following `.editorconfig` settings: + +- `indent_style` +- `indent_size` +- `end_of_line` +- `trim_trailing_whitespace` +- `tab_width` +- `insert_final_newline` + ## Commit changes -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4539) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4 and [brought to GitLab Core](https://gitlab.com/gitlab-org/gitlab-foss/issues/44157) in 10.7. -> - From [GitLab 12.7 onward](https://gitlab.com/gitlab-org/gitlab/issues/33441), files were automatically staged. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4539) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. +> - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/44157) to GitLab Core in 10.7. +> - From [GitLab 12.7 onward](https://gitlab.com/gitlab-org/gitlab/-/issues/33441), files were automatically staged. > - From [GitLab 12.9 onward](https://gitlab.com/gitlab-org/gitlab/-/issues/196609), support for staging files was removed to prevent loss of unstaged data. All your current changes necessarily have to be committed or discarded. After making your changes, click the **Commit** button on the bottom-left to @@ -116,6 +136,20 @@ in the top of the sidebar to open a list of branches. You will need to commit or discard all your changes before switching to a different branch. +## Markdown editing + +> - Markdown preview [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18059) in [GitLab Core](https://about.gitlab.com/pricing/) 10.7. +> - Support for pasting images [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22822) in [GitLab Core](https://about.gitlab.com/pricing/) 13.1. + +When you edit Markdown files in the Web IDE, you can preview your changes by +clicking the **Preview Markdown** tab above the file editor. The Markdown preview +supports [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-gfm). + +You can also upload any local images by pasting them directly in the Markdown file. +The image is uploaded to the same directory and is named `image.png` by default. +If another file already exists with the same name, a numeric suffix is automatically +added to the file name. + ## Live Preview > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19764) in [GitLab Core](https://about.gitlab.com/pricing/) 11.2. @@ -152,9 +186,10 @@ below. } ``` -## Interactive Web Terminals for the Web IDE **(ULTIMATE ONLY)** +## Interactive Web Terminals for the Web IDE -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5426) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5426) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/211685) to GitLab Core in 13.1. CAUTION: **Warning:** Interactive Web Terminals for the Web IDE is currently in **Beta**. @@ -252,7 +287,8 @@ click **Restart Terminal** to start a new terminal session. ### File syncing to web terminal -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5276) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5276) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/211686) to GitLab Core in 13.1. File changes in the Web IDE can be synced to a running web terminal. This enables users to test their code changes in a preconfigured terminal @@ -284,7 +320,7 @@ terminal: - The `webide-file-sync` executable must start **after** the project directory is available. This is why we need to add `sleep 5` to the `command`. - See [this issue](https://gitlab.com/gitlab-org/webide-file-sync/issues/7) for + See [this issue](https://gitlab.com/gitlab-org/webide-file-sync/-/issues/7) for more information. - `$CI_PROJECT_DIR` is a [predefined environment variable](../../../ci/variables/predefined_variables.md) diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index fa3ad4536ef..82dbeb0ff7e 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -52,7 +52,7 @@ When you're ready, click the **Create page** and the new page will be created. ### Attachment storage -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/33475) in GitLab 11.3. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/33475) in GitLab 11.3. Starting with GitLab 11.3, any file that is uploaded to the wiki via GitLab's interface will be stored in the wiki Git repository, and it will be available @@ -185,6 +185,14 @@ them like you would do with every other Git repository. On the right sidebar, click on **Clone repository** and follow the on-screen instructions. +Files that you add to your wiki locally must have one of the following +supported extensions, depending on the markup language you wish to use, +otherwise they will not display when pushed to GitLab: + +- Markdown extensions: `.mdown`, `.mkd`, `.mkdn`, `.md`, `.markdown`. +- AsciiDoc extensions: `.adoc`, `.ad`, `.asciidoc`. +- Other markup extensions: `.textile`, `.rdoc`, `.org`, `.creole`, `.wiki`, `.mediawiki`, `.rst`. + ## Customizing sidebar On the project's Wiki page, there is a right side navigation that renders the full Wiki pages list by default, with hierarchy. diff --git a/doc/user/search/advanced_global_search.md b/doc/user/search/advanced_global_search.md index fac1702f2ac..eaa57395b8f 100644 --- a/doc/user/search/advanced_global_search.md +++ b/doc/user/search/advanced_global_search.md @@ -2,9 +2,10 @@ > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109) in GitLab [Starter](https://about.gitlab.com/pricing/) 8.4. -NOTE: **Note** -Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. We are working on adding it. -[Follow this epic for the latest updates](https://gitlab.com/groups/gitlab-org/-/epics/153). +NOTE: **GitLab.com availability:** +Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. +It will be progressively enabled for all paid groups in Q3 of 2020. +[Follow this epic](https://gitlab.com/groups/gitlab-com/-/epics/649) for the latest updates on the timeline. Leverage Elasticsearch for faster, more advanced code search across your entire GitLab instance. diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index 5113578af9e..1ca9649b581 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -4,9 +4,10 @@ > > - Introduced in [GitLab Enterprise Starter](https://about.gitlab.com/pricing/) 9.2 -NOTE: **Note** -Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. We are working on adding it. -[Follow this epic for the latest updates](https://gitlab.com/groups/gitlab-org/-/epics/153). +NOTE: **GitLab.com availability:** +Advanced Global Search (powered by Elasticsearch) is not yet available on GitLab.com. +It will be progressively enabled for all paid groups in Q3 of 2020. +[Follow this epic](https://gitlab.com/groups/gitlab-com/-/epics/649) for the latest updates on the timeline. Use advanced queries for more targeted search results. diff --git a/doc/user/search/index.md b/doc/user/search/index.md index f5efa3519d9..b616b606b64 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -49,8 +49,8 @@ groups: 1. Select or type the operator to use for filtering the attribute. The following operators are available: - `=`: Is - - `!=`: Is not ([Introduced](https://gitlab.com/gitlab-org/gitlab/issues/18059) in GitLab 12.7) -1. Enter the text to filter the attribute by. + - `!=`: Is not ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18059) in GitLab 12.7) +1. Enter the text to [filter the attribute by](#filters-autocomplete). 1. Repeat this process to filter by multiple attributes. Multiple attributes are joined by a logical `AND`. @@ -86,7 +86,7 @@ You can filter issues and merge requests by specific terms included in titles or ### Filtering by ID -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/39908) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39908) in GitLab 12.1. You can filter the **Issues** list to individual instances by their ID. For example, enter filter `#10` to return only issue 10. The same applies to the **Merge Requests** list. Enter filter `#30` to return only merge request 30. @@ -110,6 +110,18 @@ the dropdown) **Approved-By** and select the user.  +## Filters autocomplete + +GitLab provides many filters across many pages (issues, merge requests, epics, +and pipelines among others) which you can use to narrow down your search. When +using the filter functionality, you can start typing characters to bring up +relevant users or other attributes. + +For performance optimization, there is a requirement of a minimum of three +characters to begin your search. For example, if you want to search for +issues that have the assignee "Simone Presley", you'll need to type at +least "Sim" before autocomplete gives any relevant results. + ## Search history You can view recent searches by clicking on the little arrow-clock icon, which is to the left of the search input. Click the search entry to run that search again. This feature is available for issues and merge requests. Searches are stored locally in your browser. @@ -149,9 +161,9 @@ You can search through your projects from the left menu, by clicking the menu ba On the field **Filter by name**, type the project or group name you want to find, and GitLab will filter them for you as you type. -You can also look for the projects you starred (**Starred projects**), and **Explore** all +You can also look for the projects you [starred](../project/index.md#star-a-project) (**Starred projects**), and **Explore** all public and internal projects available in GitLab.com, from which you can filter by visibility, -through **Trending**, best rated with **Most starts**, or **All** of them. +through **Trending**, best rated with **Most stars**, or **All** of them. You can also sort them by **Name**, **Last created**, **Oldest created**, **Last updated**, **Oldest updated**, **Owner**, and choose to hide or show **archived projects**: diff --git a/doc/user/shortcuts.md b/doc/user/shortcuts.md index fa466fdb3b9..efa374cf1c3 100644 --- a/doc/user/shortcuts.md +++ b/doc/user/shortcuts.md @@ -8,7 +8,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/shortcuts.html' GitLab has many useful keyboard shortcuts to make it easier to access different features. You can see a modal listing keyboard shortcuts within GitLab itself by pressing <kbd>?</kbd>, or clicking **Keyboard shortcuts** in the Help menu at the top right. -From [GitLab 12.8 onwards](https://gitlab.com/gitlab-org/gitlab/issues/22113), +In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/22113), keyboard shortcuts can be disabled using the **Enable**/**Disable** toggle in this modal window. The [Global Shortcuts](#global-shortcuts) work from any area of GitLab, but you must diff --git a/doc/user/snippets.md b/doc/user/snippets.md index 96c8dba11e5..68e5c5ac92c 100644 --- a/doc/user/snippets.md +++ b/doc/user/snippets.md @@ -129,7 +129,7 @@ different visibility level from the dropdown menu. ## Snippet comments -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/12910) in GitLab 9.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/12910) in GitLab 9.2. With GitLab Snippets you engage in a conversation about that piece of code, facilitating the collaboration among users. @@ -147,8 +147,8 @@ snippet was created using the GitLab web interface the original line ending is W > Introduced in GitLab 10.8. -Public snippets can not only be shared, but also embedded on any website. This -allows us to reuse a GitLab snippet in multiple places and any change to the source +Public snippets can not only be shared, but also embedded on any website. With +this, you can reuse a GitLab snippet in multiple places and any change to the source is automatically reflected in the embedded snippet. To embed a snippet, first make sure that: @@ -172,6 +172,6 @@ Here's how an embedded snippet looks like: <script src="https://gitlab.com/gitlab-org/gitlab-foss/snippets/1717978.js"></script> -Embedded snippets are displayed with a header that shows the file name is defined, +Embedded snippets are displayed with a header that shows the file name if it's defined, the snippet size, a link to GitLab, and the actual snippet content. Actions in the header allow users to see the snippet in raw format and download it. diff --git a/doc/user/todos.md b/doc/user/todos.md index 84a7b6afdac..5d3e3e62652 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -1,5 +1,8 @@ --- disqus_identifier: 'https://docs.gitlab.com/ee/workflow/todos.html' +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- # GitLab To-Do List |
